MirrorSave
/
budibase
mirror of https://github.com/Budibase/budibase.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Merge branch 'master' into form-block-field-layout

This commit is contained in:
Mihail Hadzhiev 2023-11-06 08:34:02 +02:00 committed by GitHub
parent cf2c8796d4 c539985088
commit d9e01ed35b
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
25 changed files with 206 additions and 193 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
.github/workflows/deploy-qa.yml vendored
View File

@ -17,6 +17,6 @@ jobs:
token: ${{ secrets.GH_ACCESS_TOKEN }}
client-payload: |-
{
"PAYLOAD_VERSION": "${{ github.sha }}",
"VERSION": "${{ github.sha }}",
"REF_NAME": "${{ github.ref_name}}"
}

4
.github/workflows/release-master.yml vendored
View File

@ -170,9 +170,9 @@ jobs:
with:
repository: budibase/budibase-deploys
event-type: budicloud-qa-deploy
github_pat: ${{ secrets.GH_ACCESS_TOKEN }}
token: ${{ secrets.GH_ACCESS_TOKEN }}
client-payload: |-
{
"PAYLOAD_VERSION": "${{ github.ref_name }}",
"VERSION": "${{ github.ref_name }}",
"REF_NAME": "${{ github.ref_name}}"
}

2
hosting/single/runner.sh
View File

@ -77,7 +77,7 @@ mkdir -p ${DATA_DIR}/minio
chown -R couchdb:couchdb ${DATA_DIR}/couch
redis-server --requirepass $REDIS_PASSWORD > /dev/stdout 2>&1 &
/bbcouch-runner.sh &
minio server --console-address ":9001" ${DATA_DIR}/minio > /dev/stdout 2>&1 &
/minio/minio server --console-address ":9001" ${DATA_DIR}/minio > /dev/stdout 2>&1 &
/etc/init.d/nginx restart
if [[ ! -z "${CUSTOM_DOMAIN}" ]]; then
# Add monthly cron job to renew certbot certificate

2
lerna.json
View File

@ -1,5 +1,5 @@
{
"version": "2.12.7",
"version": "2.12.11",
"npmClient": "yarn",
"packages": [
"packages/*"

1
packages/backend-core/src/index.ts
View File

@ -30,7 +30,6 @@ export * as timers from "./timers"
export { default as env } from "./environment"
export * as blacklist from "./blacklist"
export * as docUpdates from "./docUpdates"
export * from "./utils/Duration"
export { SearchParams } from "./db"
// Add context to tenancy for backwards compatibility
// only do this for external usages to prevent internal

6
packages/backend-core/src/objectStore/utils.ts
View File

@ -18,8 +18,12 @@ export const ObjectStoreBuckets = {
}
const bbTmp = join(tmpdir(), ".budibase")
if (!fs.existsSync(bbTmp)) {
try {
fs.mkdirSync(bbTmp)
} catch (e: any) {
if (e.code !== "EEXIST") {
throw e
}
}
export function budibaseTempDir() {

2
packages/backend-core/src/queue/inMemoryQueue.ts
View File

@ -36,7 +36,7 @@ class InMemoryQueue {
* @param opts This is not used by the in memory queue as there is no real use
* case when in memory, but is the same API as Bull
*/
constructor(name: string, opts?: any) {
constructor(name: string, opts = null) {
this._name = name
this._opts = opts
this._messages = []

20
packages/backend-core/src/queue/queue.ts
View File

@ -2,18 +2,11 @@ import env from "../environment"
import { getRedisOptions } from "../redis/utils"
import { JobQueue } from "./constants"
import InMemoryQueue from "./inMemoryQueue"
import BullQueue, { QueueOptions } from "bull"
import BullQueue from "bull"
import { addListeners, StalledFn } from "./listeners"
import { Duration } from "../utils"
import * as timers from "../timers"
import * as Redis from "ioredis"
// the queue lock is held for 5 minutes
const QUEUE_LOCK_MS = Duration.fromMinutes(5).toMs()
// queue lock is refreshed every 30 seconds
const QUEUE_LOCK_RENEW_INTERNAL_MS = Duration.fromSeconds(30).toMs()
// cleanup the queue every 60 seconds
const CLEANUP_PERIOD_MS = Duration.fromSeconds(60).toMs()
const CLEANUP_PERIOD_MS = 60 * 1000
let QUEUES: BullQueue.Queue[] | InMemoryQueue[] = []
let cleanupInterval: NodeJS.Timeout
@ -28,14 +21,7 @@ export function createQueue<T>(
opts: { removeStalledCb?: StalledFn } = {}
): BullQueue.Queue<T> {
const { opts: redisOpts, redisProtocolUrl } = getRedisOptions()
const queueConfig: QueueOptions = {
redis: redisProtocolUrl! || (redisOpts as Redis.RedisOptions),
settings: {
maxStalledCount: 0,
lockDuration: QUEUE_LOCK_MS,
lockRenewTime: QUEUE_LOCK_RENEW_INTERNAL_MS,
},
}
const queueConfig: any = redisProtocolUrl || { redis: redisOpts }
let queue: any
if (!env.isTest()) {
queue = new BullQueue(jobQueue, queueConfig)

49
packages/backend-core/src/utils/Duration.ts
View File

@ -1,49 +0,0 @@
export enum DurationType {
MILLISECONDS = "milliseconds",
SECONDS = "seconds",
MINUTES = "minutes",
HOURS = "hours",
DAYS = "days",
}
const conversion: Record<DurationType, number> = {
milliseconds: 1,
seconds: 1000,
minutes: 60 * 1000,
hours: 60 * 60 * 1000,
days: 24 * 60 * 60 * 1000,
}
export class Duration {
static convert(from: DurationType, to: DurationType, duration: number) {
const milliseconds = duration * conversion[from]
return milliseconds / conversion[to]
}
static from(from: DurationType, duration: number) {
return {
to: (to: DurationType) => {
return Duration.convert(from, to, duration)
},
toMs: () => {
return Duration.convert(from, DurationType.MILLISECONDS, duration)
},
}
}
static fromSeconds(duration: number) {
return Duration.from(DurationType.SECONDS, duration)
}
static fromMinutes(duration: number) {
return Duration.from(DurationType.MINUTES, duration)
}
static fromHours(duration: number) {
return Duration.from(DurationType.HOURS, duration)
}
static fromDays(duration: number) {
return Duration.from(DurationType.DAYS, duration)
}
}

1
packages/backend-core/src/utils/index.ts
View File

@ -1,4 +1,3 @@
export * from "./hashing"
export * from "./utils"
export * from "./stringUtils"
export * from "./Duration"

19
packages/backend-core/src/utils/tests/Duration.spec.ts
View File

@ -1,19 +0,0 @@
import { Duration, DurationType } from "../Duration"
describe("duration", () => {
it("should convert minutes to milliseconds", () => {
expect(Duration.fromMinutes(5).toMs()).toBe(300000)
})
it("should convert seconds to milliseconds", () => {
expect(Duration.fromSeconds(30).toMs()).toBe(30000)
})
it("should convert days to milliseconds", () => {
expect(Duration.fromDays(1).toMs()).toBe(86400000)
})
it("should convert minutes to days", () => {
expect(Duration.fromMinutes(1440).to(DurationType.DAYS)).toBe(1)
})
})

75
packages/frontend-core/src/components/grid/cells/RelationshipCell.svelte
View File

@ -1,27 +1,10 @@
<script context="module">
// We can create a module level cache for all relationship cells to avoid
// having to fetch the table definition one time for each cell
let primaryDisplayCache = {}
const getPrimaryDisplayForTableId = async (API, tableId) => {
if (primaryDisplayCache[tableId]) {
return primaryDisplayCache[tableId]
}
const definition = await API.fetchTableDefinition(tableId)
const primaryDisplay =
definition?.primaryDisplay || definition?.schema?.[0]?.name
primaryDisplayCache[tableId] = primaryDisplay
return primaryDisplay
}
</script>
<script>
import { getColor } from "../lib/utils"
import { onMount, getContext } from "svelte"
import { Icon, Input, ProgressCircle, clickOutside } from "@budibase/bbui"
import { debounce } from "../../../utils/utils"
const { API, dispatch } = getContext("grid")
const { API, dispatch, cache } = getContext("grid")
export let value
export let api
@ -147,7 +130,9 @@
// Find the primary display for the related table
if (!primaryDisplay) {
searching = true
primaryDisplay = await getPrimaryDisplayForTableId(API, schema.tableId)
primaryDisplay = await cache.actions.getPrimaryDisplayForTableId(
schema.tableId
)
}
// Show initial list of results
@ -195,7 +180,7 @@
const toggleRow = async row => {
if (value?.some(x => x._id === row._id)) {
// If the row is already included, remove it and update the candidate
// row to be the the same position if possible
// row to be the same position if possible
if (oneRowOnly) {
await onChange([])
} else {
@ -260,31 +245,29 @@
class:wrap={editable || contentLines > 1}
on:wheel={e => (focused ? e.stopPropagation() : null)}
>
{#if Array.isArray(value) && value.length}
{#each value as relationship}
{#if relationship[primaryDisplay] || relationship.primaryDisplay}
<div class="badge">
<span
on:click={editable
? () => showRelationship(relationship._id)
: null}
>
{readable(
relationship[primaryDisplay] || relationship.primaryDisplay
)}
</span>
{#if editable}
<Icon
name="Close"
size="XS"
hoverable
on:click={() => toggleRow(relationship)}
/>
{/if}
</div>
{/if}
{/each}
{/if}
{#each value || [] as relationship}
{#if relationship[primaryDisplay] || relationship.primaryDisplay}
<div class="badge">
<span
on:click={editable
? () => showRelationship(relationship._id)
: null}
>
{readable(
relationship[primaryDisplay] || relationship.primaryDisplay
)}
</span>
{#if editable}
<Icon
name="Close"
size="XS"
hoverable
on:click={() => toggleRow(relationship)}
/>
{/if}
</div>
{/if}
{/each}
{#if editable}
<div class="add" on:click={open}>
<Icon name="Add" size="S" />
@ -320,7 +303,7 @@
<div class="searching">
<ProgressCircle size="S" />
</div>
{:else if Array.isArray(searchResults) && searchResults.length}
{:else if searchResults?.length}
<div class="results">
{#each searchResults as row, idx}
<div

47
packages/frontend-core/src/components/grid/stores/cache.js Normal file
View File

@ -0,0 +1,47 @@
export const createActions = context => {
const { API } = context
// Cache for the primary display columns of different tables.
// If we ever need to cache table definitions for other purposes then we can
// expand this to be a more generic cache.
let primaryDisplayCache = {}
const resetPrimaryDisplayCache = () => {
primaryDisplayCache = {}
}
const getPrimaryDisplayForTableId = async tableId => {
// If we've never encountered this tableId before then store a promise that
// resolves to the primary display so that subsequent invocations before the
// promise completes can reuse this promise
if (!primaryDisplayCache[tableId]) {
primaryDisplayCache[tableId] = new Promise(resolve => {
API.fetchTableDefinition(tableId).then(def => {
const display = def?.primaryDisplay || def?.schema?.[0]?.name
primaryDisplayCache[tableId] = display
resolve(display)
})
})
}
// We await the result so that we account for both promises and primitives
return await primaryDisplayCache[tableId]
}
return {
cache: {
actions: {
getPrimaryDisplayForTableId,
resetPrimaryDisplayCache,
},
},
}
}
export const initialise = context => {
const { datasource, cache } = context
// Wipe the caches whenever the datasource changes to ensure we aren't
// storing any stale information
datasource.subscribe(cache.actions.resetPrimaryDisplayCache)
}

6
packages/frontend-core/src/components/grid/stores/datasource.js
View File

@ -160,11 +160,6 @@ export const createActions = context => {
return getAPI()?.actions.canUseColumn(name)
}
// Gets the default number of rows for a single page
const getFeatures = () => {
return getAPI()?.actions.getFeatures()
}
return {
datasource: {
...datasource,
@ -177,7 +172,6 @@ export const createActions = context => {
getRow,
isDatasourceValid,
canUseColumn,
getFeatures,
},
},
}

6
packages/frontend-core/src/components/grid/stores/datasources/nonPlus.js
View File

@ -35,11 +35,6 @@ export const createActions = context => {
return $columns.some(col => col.name === name) || $sticky?.name === name
}
const getFeatures = () => {
// We don't support any features
return {}
}
return {
nonPlus: {
actions: {
@ -50,7 +45,6 @@ export const createActions = context => {
getRow,
isDatasourceValid,
canUseColumn,
getFeatures,
},
},
}

6
packages/frontend-core/src/components/grid/stores/datasources/table.js
View File

@ -1,5 +1,4 @@
import { get } from "svelte/store"
import TableFetch from "../../../../fetch/TableFetch"
const SuppressErrors = true
@ -46,10 +45,6 @@ export const createActions = context => {
return $columns.some(col => col.name === name) || $sticky?.name === name
}
const getFeatures = () => {
return new TableFetch({ API }).determineFeatureFlags()
}
return {
table: {
actions: {
@ -60,7 +55,6 @@ export const createActions = context => {
getRow,
isDatasourceValid,
canUseColumn,
getFeatures,
},
},
}

6
packages/frontend-core/src/components/grid/stores/datasources/viewV2.js
View File

@ -1,5 +1,4 @@
import { get } from "svelte/store"
import ViewV2Fetch from "../../../../fetch/ViewV2Fetch"
const SuppressErrors = true
@ -46,10 +45,6 @@ export const createActions = context => {
)
}
const getFeatures = () => {
return new ViewV2Fetch({ API }).determineFeatureFlags()
}
return {
viewV2: {
actions: {
@ -60,7 +55,6 @@ export const createActions = context => {
getRow,
isDatasourceValid,
canUseColumn,
getFeatures,
},
},
}

2
packages/frontend-core/src/components/grid/stores/index.js
View File

@ -19,6 +19,7 @@ import * as Datasource from "./datasource"
import * as Table from "./datasources/table"
import * as ViewV2 from "./datasources/viewV2"
import * as NonPlus from "./datasources/nonPlus"
import * as Cache from "./cache"
const DependencyOrderedStores = [
Sort,
@ -42,6 +43,7 @@ const DependencyOrderedStores = [
Clipboard,
Config,
Notifications,
Cache,
]
export const attachStores = context => {

10
packages/frontend-core/src/components/grid/stores/rows.js
View File

@ -114,10 +114,6 @@ export const createActions = context => {
const $allFilters = get(allFilters)
const $sort = get(sort)
// Determine how many rows to fetch per page
const features = datasource.actions.getFeatures()
const limit = features?.supportsPagination ? RowPageSize : null
// Create new fetch model
const newFetch = fetchData({
API,
@ -126,8 +122,12 @@ export const createActions = context => {
filter: $allFilters,
sortColumn: $sort.column,
sortOrder: $sort.order,
limit,
limit: RowPageSize,
paginate: true,
// Disable client side limiting, so that for queries and custom data
// sources we don't impose fake row limits. We want all the data.
clientSideLimiting: false,
},
})

21
packages/frontend-core/src/fetch/DataFetch.js
View File

@ -43,6 +43,11 @@ export default class DataFetch {
// Pagination config
paginate: true,
// Client side feature customisation
clientSideSearching: true,
clientSideSorting: true,
clientSideLimiting: true,
}
// State of the fetch
@ -208,24 +213,32 @@ export default class DataFetch {
* Fetches some filtered, sorted and paginated data
*/
async getPage() {
const { sortColumn, sortOrder, sortType, limit } = this.options
const {
sortColumn,
sortOrder,
sortType,
limit,
clientSideSearching,
clientSideSorting,
clientSideLimiting,
} = this.options
const { query } = get(this.store)
// Get the actual data
let { rows, info, hasNextPage, cursor, error } = await this.getData()
// If we don't support searching, do a client search
if (!this.features.supportsSearch) {
if (!this.features.supportsSearch && clientSideSearching) {
rows = runLuceneQuery(rows, query)
}
// If we don't support sorting, do a client-side sort
if (!this.features.supportsSort) {
if (!this.features.supportsSort && clientSideSorting) {
rows = luceneSort(rows, sortColumn, sortOrder, sortType)
}
// If we don't support pagination, do a client-side limit
if (!this.features.supportsPagination) {
if (!this.features.supportsPagination && clientSideLimiting) {
rows = luceneLimit(rows, limit)
}

18
packages/server/src/api/routes/tests/row.spec.ts
View File

@ -49,7 +49,12 @@ describe.each([
let table: Table
let tableId: string
afterAll(setup.afterAll)
afterAll(async () => {
if (dsProvider) {
await dsProvider.stopContainer()
}
setup.afterAll()
})
beforeAll(async () => {
await config.init()
@ -521,20 +526,17 @@ describe.each([
const rowUsage = await getRowUsage()
const queryUsage = await getQueryUsage()
const res = await config.api.row.patch(table._id!, {
const row = await config.api.row.patch(table._id!, {
_id: existing._id!,
_rev: existing._rev!,
tableId: table._id!,
name: "Updated Name",
})
expect((res as any).res.statusMessage).toEqual(
`${table.name} updated successfully.`
)
expect(res.body.name).toEqual("Updated Name")
expect(res.body.description).toEqual(existing.description)
expect(row.name).toEqual("Updated Name")
expect(row.description).toEqual(existing.description)
const savedRow = await loadRow(res.body._id, table._id!)
const savedRow = await loadRow(row._id!, table._id!)
expect(savedRow.body.description).toEqual(existing.description)
expect(savedRow.body.name).toEqual("Updated Name")

61
packages/server/src/api/routes/tests/table.spec.ts
View File

@ -492,6 +492,67 @@ describe("/tables", () => {
}
})
it("should succeed when the row is created from the other side of the relationship", async () => {
// We found a bug just after releasing this feature where if the row was created from the
// users table, not the table linking to it, the migration would succeed but lose the data.
// This happened because the order of the documents in the link was reversed.
const table = await config.api.table.create({
name: "table",
type: "table",
sourceId: INTERNAL_TABLE_SOURCE_ID,
sourceType: TableSourceType.INTERNAL,
schema: {
"user relationship": {
type: FieldType.LINK,
fieldName: "test",
name: "user relationship",
constraints: {
type: "array",
presence: false,
},
relationshipType: RelationshipType.MANY_TO_ONE,
tableId: InternalTable.USER_METADATA,
},
},
})
let testRow = await config.api.row.save(table._id!, {})
await Promise.all(
users.map(u =>
config.api.row.patch(InternalTable.USER_METADATA, {
tableId: InternalTable.USER_METADATA,
_rev: u._rev!,
_id: u._id!,
test: [testRow],
})
)
)
await config.api.table.migrate(table._id!, {
oldColumn: table.schema["user relationship"],
newColumn: {
name: "user column",
type: FieldType.BB_REFERENCE,
subtype: FieldSubtype.USERS,
},
})
const migratedTable = await config.api.table.get(table._id!)
expect(migratedTable.schema["user column"]).toBeDefined()
expect(migratedTable.schema["user relationship"]).not.toBeDefined()
const resp = await config.api.row.get(table._id!, testRow._id!)
const migratedRow = resp.body as Row
expect(migratedRow["user column"]).toBeDefined()
expect(migratedRow["user relationship"]).not.toBeDefined()
expect(migratedRow["user column"]).toHaveLength(3)
expect(migratedRow["user column"].map((u: Row) => u._id)).toEqual(
expect.arrayContaining(users.map(u => u._id))
)
})
it("should successfully migrate a many-to-many user relationship to a users column", async () => {
const table = await config.api.table.create({
name: "table",

21
packages/server/src/tests/utilities/api/row.ts
View File

@ -55,7 +55,13 @@ export class RowAPI extends TestAPI {
.send(row)
.set(this.config.defaultHeaders())
.expect("Content-Type", /json/)
.expect(expectStatus)
if (resp.status !== expectStatus) {
throw new Error(
`Expected status ${expectStatus} but got ${
resp.status
}, body: ${JSON.stringify(resp.body)}`
)
}
return resp.body as Row
}
@ -77,13 +83,20 @@ export class RowAPI extends TestAPI {
sourceId: string,
row: PatchRowRequest,
{ expectStatus } = { expectStatus: 200 }
) => {
return this.request
): Promise<Row> => {
let resp = await this.request
.patch(`/api/${sourceId}/rows`)
.send(row)
.set(this.config.defaultHeaders())
.expect("Content-Type", /json/)
.expect(expectStatus)
if (resp.status !== expectStatus) {
throw new Error(
`Expected status ${expectStatus} but got ${
resp.status
}, body: ${JSON.stringify(resp.body)}`
)
}
return resp.body as Row
}
delete = async (

6
pull_request_template.md
View File

@ -1,16 +1,12 @@
## Description
_Describe the problem or feature in addition to a link to the relevant github issues._
Addresses:
### Addresses:
- `<Enter the Link to the issue(s) this PR addresses>`
- ...more if required
## App Export
- If possible, attach an app export file along with your request template to make QA testing easier, with minimal setup.
## Screenshots
_If a UI facing feature, a short video of the happy path, and some screenshots of the new functionality._

6
scripts/install-minio.sh
View File

@ -2,9 +2,9 @@
if [[ $TARGETARCH == arm* ]] ;
then
echo "INSTALLING ARM64 MINIO"
wget wget https://dl.min.io/server/minio/release/linux-arm64/archive/minio.deb -O minio.deb
wget https://dl.min.io/server/minio/release/linux-arm64/minio
else
echo "INSTALLING AMD64 MINIO"
wget wget https://dl.min.io/server/minio/release/linux-amd64/archive/minio.deb -O minio.deb
wget https://dl.min.io/server/minio/release/linux-amd64/minio
fi
dpkg -i minio.deb
chmod +x minio