MirrorSave
/
budibase
mirror of https://github.com/Budibase/budibase.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Merge branch 'master' of github.com:Budibase/budibase into feature/role-multi-inheritance

This commit is contained in:
mike12345567 2024-10-15 13:34:39 +01:00
parent bb43049d55 859982912d
commit ca72dffb1f
22 changed files with 349 additions and 88 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

46
.github/workflows/deploy-featurebranch.yml vendored
View File

@ -2,13 +2,11 @@ name: deploy-featurebranch
on:
pull_request:
types: [
labeled,
# default types below (https://docs.github.com/en/actions/using-workflows/events-that-trigger-workflows#pull_request)
opened,
synchronize,
reopened,
]
types:
- labeled
- opened
- synchronize
- reopened
jobs:
release:
@ -22,31 +20,21 @@ jobs:
contains(github.event.pull_request.labels.*.name, 'feature-branch-enterprise')
)
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v4
- name: Set PAYLOAD_LICENSE_TYPE
id: set_license_type
run: |
if [[ "${{ contains(github.event.pull_request.labels.*.name, 'feature-branch') }}" == "true" ]]; then
echo "PAYLOAD_LICENSE_TYPE=free" >> $GITHUB_ENV
elif [[ "${{ contains(github.event.pull_request.labels.*.name, 'feature-branch-pro') }}" == "true" ]]; then
echo "PAYLOAD_LICENSE_TYPE=pro" >> $GITHUB_ENV
elif [[ "${{ contains(github.event.pull_request.labels.*.name, 'feature-branch-team') }}" == "true" ]]; then
echo "PAYLOAD_LICENSE_TYPE=team" >> $GITHUB_ENV
elif [[ "${{ contains(github.event.pull_request.labels.*.name, 'feature-branch-business') }}" == "true" ]]; then
echo "PAYLOAD_LICENSE_TYPE=business" >> $GITHUB_ENV
elif [[ "${{ contains(github.event.pull_request.labels.*.name, 'feature-branch-enterprise') }}" == "true" ]]; then
echo "PAYLOAD_LICENSE_TYPE=enterprise" >> $GITHUB_ENV
else
echo "PAYLOAD_LICENSE_TYPE=free" >> $GITHUB_ENV
fi
- uses: passeidireto/trigger-external-workflow-action@main
env:
PAYLOAD_BRANCH: ${{ github.head_ref }}
PAYLOAD_PR_NUMBER: ${{ github.event.pull_request.number }}
PAYLOAD_LICENSE_TYPE: ${{ env.PAYLOAD_LICENSE_TYPE }}
PAYLOAD_LICENSE_TYPE: |
${{
contains(github.event.pull_request.labels.*.name, 'feature-branch') && 'free' ||
contains(github.event.pull_request.labels.*.name, 'feature-branch-pro') && 'pro' ||
contains(github.event.pull_request.labels.*.name, 'feature-branch-team') && 'team' ||
contains(github.event.pull_request.labels.*.name, 'feature-branch-business') && 'business' ||
contains(github.event.pull_request.labels.*.name, 'feature-branch-enterprise') && 'enterprise' || 'free'
}}
steps:
- uses: actions/checkout@v4
- uses: passeidireto/trigger-external-workflow-action@main
with:
repository: budibase/budibase-deploys
event: featurebranch-qa-deploy

4
charts/budibase/templates/app-service-deployment.yaml
View File

@ -42,14 +42,12 @@ spec:
{{ else }}
value: http://{{ .Release.Name }}-svc-couchdb:{{ .Values.services.couchdb.port }}
{{ end }}
{{ if .Values.globals.sqs.enabled }}
- name: COUCH_DB_SQL_URL
{{ if .Values.globals.sqs.url }}
value: {{ .Values.globals.sqs.url }}
value: {{ .Values.globals.sqs.url | quote }}
{{ else }}
value: http://{{ .Release.Name }}-svc-couchdb:{{ .Values.globals.sqs.port }}
{{ end }}
{{ end }}
{{ if .Values.services.couchdb.enabled }}
- name: COUCH_DB_USER
valueFrom:

6
charts/budibase/templates/automation-worker-service-deployment.yaml
View File

@ -43,6 +43,12 @@ spec:
{{ else }}
value: http://{{ .Release.Name }}-svc-couchdb:{{ .Values.services.couchdb.port }}
{{ end }}
- name: COUCH_DB_SQL_URL
{{ if .Values.globals.sqs.url }}
value: {{ .Values.globals.sqs.url | quote }}
{{ else }}
value: http://{{ .Release.Name }}-svc-couchdb:{{ .Values.globals.sqs.port }}
{{ end }}
{{ if .Values.services.couchdb.enabled }}
- name: COUCH_DB_USER
valueFrom:

4
charts/budibase/templates/worker-service-deployment.yaml
View File

@ -56,14 +56,12 @@ spec:
{{ else }}
value: http://{{ .Release.Name }}-svc-couchdb:{{ .Values.services.couchdb.port }}
{{ end }}
{{ if .Values.globals.sqs.enabled }}
- name: COUCH_DB_SQL_URL
{{ if .Values.globals.sqs.url }}
value: {{ .Values.globals.sqs.url }}
value: {{ .Values.globals.sqs.url | quote }}
{{ else }}
value: http://{{ .Release.Name }}-svc-couchdb:{{ .Values.globals.sqs.port }}
{{ end }}
{{ end }}
- name: API_ENCRYPTION_KEY
valueFrom:
secretKeyRef:

3
charts/budibase/values.yaml
View File

@ -139,9 +139,6 @@ globals:
password: ""
sqs:
# -- Whether to use the CouchDB "structured query service" or not. This is disabled by
# default for now, but will become the default in a future release.
enabled: false
# @ignore
url: ""
# @ignore

6
hosting/docker-compose.yaml
View File

@ -5,7 +5,7 @@ version: "3"
services:
app-service:
restart: unless-stopped
image: budibase.docker.scarf.sh/budibase/apps
image: budibase/apps
container_name: bbapps
environment:
SELF_HOSTED: 1
@ -35,7 +35,7 @@ services:
worker-service:
restart: unless-stopped
image: budibase.docker.scarf.sh/budibase/worker
image: budibase/worker
container_name: bbworker
environment:
SELF_HOSTED: 1
@ -97,7 +97,7 @@ services:
couchdb-service:
restart: unless-stopped
image: budibase/couchdb
image: budibase/couchdb:v3.3.3-sqs-v2.1.1
environment:
- COUCHDB_PASSWORD=${COUCH_DB_PASSWORD}
- COUCHDB_USER=${COUCH_DB_USER}

3
hosting/single/Dockerfile
View File

@ -69,6 +69,9 @@ WORKDIR /minio
COPY scripts/install-minio.sh ./install.sh
RUN chmod +x install.sh && ./install.sh
# setup redis
COPY hosting/single/redis.conf /etc/redis/redis.conf
# setup runner file
WORKDIR /
COPY hosting/single/runner.sh .

7
hosting/single/redis.conf Normal file
View File

@ -0,0 +1,7 @@
dir "DATA_DIR/redis"
appendonly yes
appendfsync everysec
auto-aof-rewrite-percentage 100
auto-aof-rewrite-min-size 64mb

8
hosting/single/runner.sh
View File

@ -75,13 +75,17 @@ fi
for LINE in $(cat ${DATA_DIR}/.env); do export $LINE; done
ln -s ${DATA_DIR}/.env /app/.env
ln -s ${DATA_DIR}/.env /worker/.env
# make these directories in runner, incase of mount
mkdir -p ${DATA_DIR}/minio
mkdir -p ${DATA_DIR}/redis
chown -R couchdb:couchdb ${DATA_DIR}/couch
sed -i "s#DATA_DIR#${DATA_DIR}#g" /etc/redis/redis.conf
if [[ -n "${REDIS_PASSWORD}" ]]; then
redis-server --requirepass $REDIS_PASSWORD > /dev/stdout 2>&1 &
redis-server /etc/redis/redis.conf --requirepass $REDIS_PASSWORD > /dev/stdout 2>&1 &
else
redis-server > /dev/stdout 2>&1 &
redis-server /etc/redis/redis.conf > /dev/stdout 2>&1 &
fi
/bbcouch-runner.sh &

2
packages/backend-core/src/features/features.ts
View File

@ -269,7 +269,7 @@ export class FlagSet<V extends Flag<any>, T extends { [key: string]: V }> {
export const flags = new FlagSet({
DEFAULT_VALUES: Flag.boolean(env.isDev()),
AUTOMATION_BRANCHING: Flag.boolean(env.isDev()),
SQS: Flag.boolean(env.isDev()),
SQS: Flag.boolean(true),
[FeatureFlag.AI_CUSTOM_CONFIGS]: Flag.boolean(env.isDev()),
[FeatureFlag.ENRICHED_RELATIONSHIPS]: Flag.boolean(env.isDev()),
[FeatureFlag.TABLES_DEFAULT_ADMIN]: Flag.boolean(env.isDev()),

6
packages/backend-core/src/features/tests/features.spec.ts
View File

@ -10,6 +10,7 @@ const schema = {
TEST_BOOLEAN: Flag.boolean(false),
TEST_STRING: Flag.string("default value"),
TEST_NUMBER: Flag.number(0),
TEST_BOOLEAN_DEFAULT_TRUE: Flag.boolean(true),
}
const flags = new FlagSet(schema)
@ -123,6 +124,11 @@ describe("feature flags", () => {
},
expected: flags.defaults(),
},
{
it: "should be possible to override a default true flag to false",
environmentFlags: "default:!TEST_BOOLEAN_DEFAULT_TRUE",
expected: { TEST_BOOLEAN_DEFAULT_TRUE: false },
},
])(
"$it",
async ({

8
packages/backend-core/src/sql/sql.ts
View File

@ -521,8 +521,11 @@ class InternalBuilder {
const [filterTableName, ...otherProperties] = key.split(".")
const property = otherProperties.join(".")
const alias = getTableAlias(filterTableName)
return fn(q, alias ? `${alias}.${property}` : property, value)
return q.andWhere(subquery =>
fn(subquery, alias ? `${alias}.${property}` : property, value)
)
}
for (const key in structure) {
const value = structure[key]
const updatedKey = dbCore.removeKeyNumbering(key)
@ -552,6 +555,9 @@ class InternalBuilder {
value
)
} else if (shouldProcessRelationship) {
if (allOr) {
query = query.or
}
query = builder.addRelationshipForFilter(query, updatedKey, q => {
return handleRelationship(q, updatedKey, value)
})

1
packages/builder/src/pages/builder/app/[application]/data/datasource/[datasourceId]/_components/panels/Tables/CreateExternalTableModal.svelte
View File

@ -23,6 +23,7 @@
sourceType: DB_TYPE_EXTERNAL,
schema: {
id: {
name: "id",
autocolumn: true,
type: "number",
},

37
packages/frontend-core/src/components/Updating.svelte
View File

@ -2,36 +2,31 @@
export let isMigrationDone
export let onMigrationDone
export let timeoutSeconds = 60 // 1 minute
export let minTimeSeconds = 3
const loadTime = Date.now()
const intervalMs = 1000
let timedOut = false
let secondsWaited = 0
async function checkMigrationsFinished() {
setTimeout(async () => {
let totalWaitMs = 0
// eslint-disable-next-line no-constant-condition
while (true) {
const waitForMs = 5000 + Math.random() * 5000
await new Promise(resolve => setTimeout(resolve, waitForMs))
totalWaitMs += waitForMs
const isMigrated = await isMigrationDone()
const timeoutMs = timeoutSeconds * 1000
if (!isMigrated || secondsWaited <= minTimeSeconds) {
if (loadTime + timeoutMs > Date.now()) {
secondsWaited += 1
return checkMigrationsFinished()
}
return migrationTimeout()
}
if (isMigrated) {
onMigrationDone()
}, intervalMs)
return
}
if (totalWaitMs > timeoutSeconds * 1000) {
timedOut = true
return
}
}
}
checkMigrationsFinished()
function migrationTimeout() {
timedOut = true
}
</script>
<div class="loading" class:timeout={timedOut}>

15
packages/server/src/api/controllers/table/index.ts
View File

@ -71,19 +71,20 @@ export async function fetch(ctx: UserCtx<void, FetchTablesResponse>) {
const datasources = await sdk.datasources.getExternalDatasources()
const external = datasources.flatMap(datasource => {
const external: Table[] = []
for (const datasource of datasources) {
let entities = datasource.entities
if (entities) {
return Object.values(entities).map<Table>((entity: Table) => ({
...entity,
for (const entity of Object.values(entities)) {
external.push({
...(await processTable(entity)),
sourceType: TableSourceType.EXTERNAL,
sourceId: datasource._id!,
sql: isSQL(datasource),
}))
} else {
return []
}
})
}
}
}
const result: FetchTablesResponse = []
for (const table of [...internal, ...external]) {

33
packages/server/src/api/routes/tests/row.spec.ts
View File

@ -789,6 +789,39 @@ describe.each([
})
})
describe("multi-user column", () => {
beforeAll(async () => {
table = await config.api.table.save(
saveTableRequest({
schema: {
users: {
name: "users",
type: FieldType.BB_REFERENCE,
subtype: BBReferenceFieldSubType.USER,
default: ["{{ [Current User]._id }}"],
},
},
})
)
})
it("creates a new row with a default value successfully", async () => {
const row = await config.api.row.save(table._id!, {})
expect(row.users).toHaveLength(1)
expect(row.users[0]._id).toEqual(config.getUser()._id)
})
it("does not use default value if value specified", async () => {
const id = `us_${utils.newid()}`
await config.createUser({ _id: id })
const row = await config.api.row.save(table._id!, {
users: [id],
})
expect(row.users).toHaveLength(1)
expect(row.users[0]._id).toEqual(id)
})
})
describe("bindings", () => {
describe("string column", () => {
beforeAll(async () => {

206
packages/server/src/api/routes/tests/search.spec.ts
View File

@ -23,6 +23,7 @@ import {
EmptyFilterOption,
FieldType,
JsonFieldSubType,
LogicalOperator,
RelationshipType,
Row,
RowSearchParams,
@ -2329,6 +2330,211 @@ describe.each([
equal: { ["name"]: "baz" },
}).toContainExactly([{ name: "baz", productCat: undefined }])
})
describe("logical filters", () => {
const logicalOperators = [LogicalOperator.AND, LogicalOperator.OR]
describe("$and", () => {
it("should allow single conditions", async () => {
await expectQuery({
$and: {
conditions: [
{
equal: { ["productCat.name"]: "foo" },
},
],
},
}).toContainExactly([
{ name: "foo", productCat: [{ _id: productCatRows[0]._id }] },
])
})
it("should allow exclusive conditions", async () => {
await expectQuery({
$and: {
conditions: [
{
equal: { ["productCat.name"]: "foo" },
notEqual: { ["productCat.name"]: "foo" },
},
],
},
}).toContainExactly([])
})
it.each([logicalOperators])(
"should allow nested ands with single conditions (with %s as root)",
async rootOperator => {
await expectQuery({
[rootOperator]: {
conditions: [
{
$and: {
conditions: [
{
equal: { ["productCat.name"]: "foo" },
},
],
},
},
],
},
}).toContainExactly([
{ name: "foo", productCat: [{ _id: productCatRows[0]._id }] },
])
}
)
it.each([logicalOperators])(
"should allow nested ands with exclusive conditions (with %s as root)",
async rootOperator => {
await expectQuery({
[rootOperator]: {
conditions: [
{
$and: {
conditions: [
{
equal: { ["productCat.name"]: "foo" },
notEqual: { ["productCat.name"]: "foo" },
},
],
},
},
],
},
}).toContainExactly([])
}
)
it.each([logicalOperators])(
"should allow nested ands with multiple conditions (with %s as root)",
async rootOperator => {
await expectQuery({
[rootOperator]: {
conditions: [
{
$and: {
conditions: [
{
equal: { ["productCat.name"]: "foo" },
},
],
},
notEqual: { ["productCat.name"]: "foo" },
},
],
},
}).toContainExactly([])
}
)
})
describe("$ors", () => {
it("should allow single conditions", async () => {
await expectQuery({
$or: {
conditions: [
{
equal: { ["productCat.name"]: "foo" },
},
],
},
}).toContainExactly([
{ name: "foo", productCat: [{ _id: productCatRows[0]._id }] },
])
})
it("should allow exclusive conditions", async () => {
await expectQuery({
$or: {
conditions: [
{
equal: { ["productCat.name"]: "foo" },
notEqual: { ["productCat.name"]: "foo" },
},
],
},
}).toContainExactly([
{ name: "foo", productCat: [{ _id: productCatRows[0]._id }] },
{ name: "bar", productCat: [{ _id: productCatRows[1]._id }] },
// { name: "baz", productCat: undefined }, // TODO
])
})
it.each([logicalOperators])(
"should allow nested ors with single conditions (with %s as root)",
async rootOperator => {
await expectQuery({
[rootOperator]: {
conditions: [
{
$or: {
conditions: [
{
equal: { ["productCat.name"]: "foo" },
},
],
},
},
],
},
}).toContainExactly([
{ name: "foo", productCat: [{ _id: productCatRows[0]._id }] },
])
}
)
it.each([logicalOperators])(
"should allow nested ors with exclusive conditions (with %s as root)",
async rootOperator => {
await expectQuery({
[rootOperator]: {
conditions: [
{
$or: {
conditions: [
{
equal: { ["productCat.name"]: "foo" },
notEqual: { ["productCat.name"]: "foo" },
},
],
},
},
],
},
}).toContainExactly([
{ name: "foo", productCat: [{ _id: productCatRows[0]._id }] },
{ name: "bar", productCat: [{ _id: productCatRows[1]._id }] },
// { name: "baz", productCat: undefined }, // TODO
])
}
)
it("should allow nested ors with multiple conditions", async () => {
await expectQuery({
$or: {
conditions: [
{
$or: {
conditions: [
{
equal: { ["productCat.name"]: "foo" },
},
],
},
notEqual: { ["productCat.name"]: "foo" },
},
],
},
}).toContainExactly([
{ name: "foo", productCat: [{ _id: productCatRows[0]._id }] },
{ name: "bar", productCat: [{ _id: productCatRows[1]._id }] },
// { name: "baz", productCat: undefined }, // TODO
])
})
})
})
})
isSql &&

4
packages/server/src/integrations/tests/sqlAlias.spec.ts
View File

@ -79,7 +79,7 @@ describe("Captures of real examples", () => {
sql: expect.stringContaining(
multiline(
`where exists (select 1 from "tasks" as "b" inner join "products_tasks" as "c" on "b"."taskid" = "c"."taskid" where "c"."productid" = "a"."productid"
and COALESCE("b"."taskname" = $1, FALSE)`
and (COALESCE("b"."taskname" = $1, FALSE))`
)
),
})
@ -144,7 +144,7 @@ describe("Captures of real examples", () => {
],
sql: expect.stringContaining(
multiline(
`where exists (select 1 from "persons" as "c" where "c"."personid" = "a"."executorid" and "c"."year" between $1 and $2)`
`where exists (select 1 from "persons" as "c" where "c"."personid" = "a"."executorid" and ("c"."year" between $1 and $2))`
)
),
})

6
packages/server/src/sdk/app/tables/getters.ts
View File

@ -20,7 +20,13 @@ export async function processTable(table: Table): Promise<Table> {
if (!table) {
return table
}
table = { ...table }
if (table._id && isExternalTableID(table._id)) {
// Old created external tables via Budibase might have a missing field name breaking some UI such as filters
if (table.schema["id"] && !table.schema["id"].name) {
table.schema["id"].name = "id"
}
return {
...table,
type: "table",

15
packages/server/src/utilities/rowProcessor/index.ts
View File

@ -33,7 +33,7 @@ import {
PROTECTED_EXTERNAL_COLUMNS,
PROTECTED_INTERNAL_COLUMNS,
} from "@budibase/shared-core"
import { processString } from "@budibase/string-templates"
import { processStringSync } from "@budibase/string-templates"
import {
getTableFromSource,
isUserMetadataTable,
@ -134,10 +134,15 @@ async function processDefaultValues(table: Table, row: Row) {
for (const [key, schema] of Object.entries(table.schema)) {
if ("default" in schema && schema.default != null && row[key] == null) {
const processed =
typeof schema.default === "string"
? await processString(schema.default, ctx)
: schema.default
let processed: string | string[]
if (Array.isArray(schema.default)) {
processed = schema.default.map(val => processStringSync(val, ctx))
} else if (typeof schema.default === "string") {
processed = processStringSync(schema.default, ctx)
} else {
processed = schema.default
}
try {
row[key] = coerce(processed, schema.type)
} catch (err: any) {

2
packages/shared-core/src/table.ts
View File

@ -66,7 +66,7 @@ const allowDefaultColumnByType: Record<FieldType, boolean> = {
[FieldType.ATTACHMENT_SINGLE]: false,
[FieldType.SIGNATURE_SINGLE]: false,
[FieldType.LINK]: false,
[FieldType.BB_REFERENCE]: false,
[FieldType.BB_REFERENCE]: true,
[FieldType.BB_REFERENCE_SINGLE]: true,
}

1
packages/types/src/documents/app/table/schema.ts
View File

@ -121,6 +121,7 @@ export interface BBReferenceFieldMetadata
type: FieldType.BB_REFERENCE
subtype: BBReferenceFieldSubType
relationshipType?: RelationshipType
default?: string[]
}
export interface BBReferenceSingleFieldMetadata
extends Omit<BaseFieldSchema, "subtype"> {