diff --git a/.github/workflows/deploy-featurebranch.yml b/.github/workflows/deploy-featurebranch.yml index 0e19f0649f..872faa98fa 100644 --- a/.github/workflows/deploy-featurebranch.yml +++ b/.github/workflows/deploy-featurebranch.yml @@ -2,13 +2,11 @@ name: deploy-featurebranch on: pull_request: - types: [ - labeled, - # default types below (https://docs.github.com/en/actions/using-workflows/events-that-trigger-workflows#pull_request) - opened, - synchronize, - reopened, - ] + types: + - labeled + - opened + - synchronize + - reopened jobs: release: @@ -22,31 +20,21 @@ jobs: contains(github.event.pull_request.labels.*.name, 'feature-branch-enterprise') ) runs-on: ubuntu-latest + env: + PAYLOAD_BRANCH: ${{ github.head_ref }} + PAYLOAD_PR_NUMBER: ${{ github.event.pull_request.number }} + PAYLOAD_LICENSE_TYPE: | + ${{ + contains(github.event.pull_request.labels.*.name, 'feature-branch') && 'free' || + contains(github.event.pull_request.labels.*.name, 'feature-branch-pro') && 'pro' || + contains(github.event.pull_request.labels.*.name, 'feature-branch-team') && 'team' || + contains(github.event.pull_request.labels.*.name, 'feature-branch-business') && 'business' || + contains(github.event.pull_request.labels.*.name, 'feature-branch-enterprise') && 'enterprise' || 'free' + }} steps: - uses: actions/checkout@v4 - - name: Set PAYLOAD_LICENSE_TYPE - id: set_license_type - run: | - if [[ "${{ contains(github.event.pull_request.labels.*.name, 'feature-branch') }}" == "true" ]]; then - echo "PAYLOAD_LICENSE_TYPE=free" >> $GITHUB_ENV - elif [[ "${{ contains(github.event.pull_request.labels.*.name, 'feature-branch-pro') }}" == "true" ]]; then - echo "PAYLOAD_LICENSE_TYPE=pro" >> $GITHUB_ENV - elif [[ "${{ contains(github.event.pull_request.labels.*.name, 'feature-branch-team') }}" == "true" ]]; then - echo "PAYLOAD_LICENSE_TYPE=team" >> $GITHUB_ENV - elif [[ "${{ contains(github.event.pull_request.labels.*.name, 'feature-branch-business') }}" == "true" ]]; then - echo "PAYLOAD_LICENSE_TYPE=business" >> $GITHUB_ENV - elif [[ "${{ contains(github.event.pull_request.labels.*.name, 'feature-branch-enterprise') }}" == "true" ]]; then - echo "PAYLOAD_LICENSE_TYPE=enterprise" >> $GITHUB_ENV - else - echo "PAYLOAD_LICENSE_TYPE=free" >> $GITHUB_ENV - fi - - uses: passeidireto/trigger-external-workflow-action@main - env: - PAYLOAD_BRANCH: ${{ github.head_ref }} - PAYLOAD_PR_NUMBER: ${{ github.event.pull_request.number }} - PAYLOAD_LICENSE_TYPE: ${{ env.PAYLOAD_LICENSE_TYPE }} with: repository: budibase/budibase-deploys event: featurebranch-qa-deploy diff --git a/charts/budibase/templates/app-service-deployment.yaml b/charts/budibase/templates/app-service-deployment.yaml index 4d0560312f..278bd1767f 100644 --- a/charts/budibase/templates/app-service-deployment.yaml +++ b/charts/budibase/templates/app-service-deployment.yaml @@ -42,14 +42,12 @@ spec: {{ else }} value: http://{{ .Release.Name }}-svc-couchdb:{{ .Values.services.couchdb.port }} {{ end }} - {{ if .Values.globals.sqs.enabled }} - name: COUCH_DB_SQL_URL - {{ if .Values.globals.sqs.url }} - value: {{ .Values.globals.sqs.url }} - {{ else }} + {{ if .Values.globals.sqs.url }} + value: {{ .Values.globals.sqs.url | quote }} + {{ else }} value: http://{{ .Release.Name }}-svc-couchdb:{{ .Values.globals.sqs.port }} - {{ end }} - {{ end }} + {{ end }} {{ if .Values.services.couchdb.enabled }} - name: COUCH_DB_USER valueFrom: diff --git a/charts/budibase/templates/automation-worker-service-deployment.yaml b/charts/budibase/templates/automation-worker-service-deployment.yaml index 71089bd7ee..e0d43d0ce6 100644 --- a/charts/budibase/templates/automation-worker-service-deployment.yaml +++ b/charts/budibase/templates/automation-worker-service-deployment.yaml @@ -43,6 +43,12 @@ spec: {{ else }} value: http://{{ .Release.Name }}-svc-couchdb:{{ .Values.services.couchdb.port }} {{ end }} + - name: COUCH_DB_SQL_URL + {{ if .Values.globals.sqs.url }} + value: {{ .Values.globals.sqs.url | quote }} + {{ else }} + value: http://{{ .Release.Name }}-svc-couchdb:{{ .Values.globals.sqs.port }} + {{ end }} {{ if .Values.services.couchdb.enabled }} - name: COUCH_DB_USER valueFrom: diff --git a/charts/budibase/templates/worker-service-deployment.yaml b/charts/budibase/templates/worker-service-deployment.yaml index dcab33fa58..94fdd0b94e 100644 --- a/charts/budibase/templates/worker-service-deployment.yaml +++ b/charts/budibase/templates/worker-service-deployment.yaml @@ -56,14 +56,12 @@ spec: {{ else }} value: http://{{ .Release.Name }}-svc-couchdb:{{ .Values.services.couchdb.port }} {{ end }} - {{ if .Values.globals.sqs.enabled }} - name: COUCH_DB_SQL_URL - {{ if .Values.globals.sqs.url }} - value: {{ .Values.globals.sqs.url }} - {{ else }} + {{ if .Values.globals.sqs.url }} + value: {{ .Values.globals.sqs.url | quote }} + {{ else }} value: http://{{ .Release.Name }}-svc-couchdb:{{ .Values.globals.sqs.port }} - {{ end }} - {{ end }} + {{ end }} - name: API_ENCRYPTION_KEY valueFrom: secretKeyRef: diff --git a/charts/budibase/values.yaml b/charts/budibase/values.yaml index 2c1525bd90..de2cdb9474 100644 --- a/charts/budibase/values.yaml +++ b/charts/budibase/values.yaml @@ -139,9 +139,6 @@ globals: password: "" sqs: - # -- Whether to use the CouchDB "structured query service" or not. This is disabled by - # default for now, but will become the default in a future release. - enabled: false # @ignore url: "" # @ignore diff --git a/hosting/docker-compose.yaml b/hosting/docker-compose.yaml index a72b36aef1..c7a22eb2b3 100644 --- a/hosting/docker-compose.yaml +++ b/hosting/docker-compose.yaml @@ -5,7 +5,7 @@ version: "3" services: app-service: restart: unless-stopped - image: budibase.docker.scarf.sh/budibase/apps + image: budibase/apps container_name: bbapps environment: SELF_HOSTED: 1 @@ -35,7 +35,7 @@ services: worker-service: restart: unless-stopped - image: budibase.docker.scarf.sh/budibase/worker + image: budibase/worker container_name: bbworker environment: SELF_HOSTED: 1 @@ -97,7 +97,7 @@ services: couchdb-service: restart: unless-stopped - image: budibase/couchdb + image: budibase/couchdb:v3.3.3-sqs-v2.1.1 environment: - COUCHDB_PASSWORD=${COUCH_DB_PASSWORD} - COUCHDB_USER=${COUCH_DB_USER} diff --git a/hosting/single/Dockerfile b/hosting/single/Dockerfile index ded0bc17dc..a1230f3c37 100644 --- a/hosting/single/Dockerfile +++ b/hosting/single/Dockerfile @@ -69,6 +69,9 @@ WORKDIR /minio COPY scripts/install-minio.sh ./install.sh RUN chmod +x install.sh && ./install.sh +# setup redis +COPY hosting/single/redis.conf /etc/redis/redis.conf + # setup runner file WORKDIR / COPY hosting/single/runner.sh . diff --git a/hosting/single/redis.conf b/hosting/single/redis.conf new file mode 100644 index 0000000000..00740ffece --- /dev/null +++ b/hosting/single/redis.conf @@ -0,0 +1,7 @@ +dir "DATA_DIR/redis" + +appendonly yes +appendfsync everysec + +auto-aof-rewrite-percentage 100 +auto-aof-rewrite-min-size 64mb \ No newline at end of file diff --git a/hosting/single/runner.sh b/hosting/single/runner.sh index 95464dd031..d9b8719f0f 100644 --- a/hosting/single/runner.sh +++ b/hosting/single/runner.sh @@ -75,13 +75,17 @@ fi for LINE in $(cat ${DATA_DIR}/.env); do export $LINE; done ln -s ${DATA_DIR}/.env /app/.env ln -s ${DATA_DIR}/.env /worker/.env + # make these directories in runner, incase of mount mkdir -p ${DATA_DIR}/minio +mkdir -p ${DATA_DIR}/redis chown -R couchdb:couchdb ${DATA_DIR}/couch + +sed -i "s#DATA_DIR#${DATA_DIR}#g" /etc/redis/redis.conf if [[ -n "${REDIS_PASSWORD}" ]]; then - redis-server --requirepass $REDIS_PASSWORD > /dev/stdout 2>&1 & + redis-server /etc/redis/redis.conf --requirepass $REDIS_PASSWORD > /dev/stdout 2>&1 & else - redis-server > /dev/stdout 2>&1 & + redis-server /etc/redis/redis.conf > /dev/stdout 2>&1 & fi /bbcouch-runner.sh & diff --git a/lerna.json b/lerna.json index 13530e9aee..caaa07683a 100644 --- a/lerna.json +++ b/lerna.json @@ -1,6 +1,6 @@ { "$schema": "node_modules/lerna/schemas/lerna-schema.json", - "version": "2.32.17", + "version": "2.33.1", "npmClient": "yarn", "packages": [ "packages/*", diff --git a/packages/backend-core/src/constants/misc.ts b/packages/backend-core/src/constants/misc.ts index aee099e10a..e2fd975e40 100644 --- a/packages/backend-core/src/constants/misc.ts +++ b/packages/backend-core/src/constants/misc.ts @@ -28,6 +28,7 @@ export enum Config { OIDC = "oidc", OIDC_LOGOS = "logos_oidc", SCIM = "scim", + AI = "AI", } export const MIN_VALID_DATE = new Date(-2147483647000) diff --git a/packages/backend-core/src/features/features.ts b/packages/backend-core/src/features/features.ts index 20b207bb02..e95472a784 100644 --- a/packages/backend-core/src/features/features.ts +++ b/packages/backend-core/src/features/features.ts @@ -269,7 +269,7 @@ export class FlagSet, T extends { [key: string]: V }> { export const flags = new FlagSet({ DEFAULT_VALUES: Flag.boolean(env.isDev()), AUTOMATION_BRANCHING: Flag.boolean(env.isDev()), - SQS: Flag.boolean(env.isDev()), + SQS: Flag.boolean(true), [FeatureFlag.AI_CUSTOM_CONFIGS]: Flag.boolean(env.isDev()), [FeatureFlag.ENRICHED_RELATIONSHIPS]: Flag.boolean(env.isDev()), [FeatureFlag.TABLES_DEFAULT_ADMIN]: Flag.boolean(env.isDev()), diff --git a/packages/backend-core/src/features/tests/features.spec.ts b/packages/backend-core/src/features/tests/features.spec.ts index 9af8a8f4bb..ced874f4af 100644 --- a/packages/backend-core/src/features/tests/features.spec.ts +++ b/packages/backend-core/src/features/tests/features.spec.ts @@ -10,6 +10,7 @@ const schema = { TEST_BOOLEAN: Flag.boolean(false), TEST_STRING: Flag.string("default value"), TEST_NUMBER: Flag.number(0), + TEST_BOOLEAN_DEFAULT_TRUE: Flag.boolean(true), } const flags = new FlagSet(schema) @@ -123,6 +124,11 @@ describe("feature flags", () => { }, expected: flags.defaults(), }, + { + it: "should be possible to override a default true flag to false", + environmentFlags: "default:!TEST_BOOLEAN_DEFAULT_TRUE", + expected: { TEST_BOOLEAN_DEFAULT_TRUE: false }, + }, ])( "$it", async ({ diff --git a/packages/backend-core/src/sql/sql.ts b/packages/backend-core/src/sql/sql.ts index 382eca3f76..8ff3fb1606 100644 --- a/packages/backend-core/src/sql/sql.ts +++ b/packages/backend-core/src/sql/sql.ts @@ -521,8 +521,11 @@ class InternalBuilder { const [filterTableName, ...otherProperties] = key.split(".") const property = otherProperties.join(".") const alias = getTableAlias(filterTableName) - return fn(q, alias ? `${alias}.${property}` : property, value) + return q.andWhere(subquery => + fn(subquery, alias ? `${alias}.${property}` : property, value) + ) } + for (const key in structure) { const value = structure[key] const updatedKey = dbCore.removeKeyNumbering(key) @@ -552,6 +555,9 @@ class InternalBuilder { value ) } else if (shouldProcessRelationship) { + if (allOr) { + query = query.or + } query = builder.addRelationshipForFilter(query, updatedKey, q => { return handleRelationship(q, updatedKey, value) }) @@ -1288,7 +1294,8 @@ class InternalBuilder { schema.constraints?.presence === true || schema.type === FieldType.FORMULA || schema.type === FieldType.AUTO || - schema.type === FieldType.LINK + schema.type === FieldType.LINK || + schema.type === FieldType.AI ) { continue } diff --git a/packages/backend-core/src/sql/sqlTable.ts b/packages/backend-core/src/sql/sqlTable.ts index f5b02cc4e4..84f4e290aa 100644 --- a/packages/backend-core/src/sql/sqlTable.ts +++ b/packages/backend-core/src/sql/sqlTable.ts @@ -17,7 +17,7 @@ import SchemaBuilder = Knex.SchemaBuilder import CreateTableBuilder = Knex.CreateTableBuilder function isIgnoredType(type: FieldType) { - const ignored = [FieldType.LINK, FieldType.FORMULA] + const ignored = [FieldType.LINK, FieldType.FORMULA, FieldType.AI] return ignored.indexOf(type) !== -1 } @@ -144,6 +144,9 @@ function generateSchema( case FieldType.FORMULA: // This is allowed, but nothing to do on the external datasource break + case FieldType.AI: + // This is allowed, but nothing to do on the external datasource + break case FieldType.ATTACHMENTS: case FieldType.ATTACHMENT_SINGLE: case FieldType.SIGNATURE_SINGLE: diff --git a/packages/backend-core/tests/core/utilities/mocks/licenses.ts b/packages/backend-core/tests/core/utilities/mocks/licenses.ts index bc9a3b635c..5ba6fb36a1 100644 --- a/packages/backend-core/tests/core/utilities/mocks/licenses.ts +++ b/packages/backend-core/tests/core/utilities/mocks/licenses.ts @@ -102,6 +102,14 @@ export const useAppBuilders = () => { return useFeature(Feature.APP_BUILDERS) } +export const useBudibaseAI = () => { + return useFeature(Feature.BUDIBASE_AI) +} + +export const useAICustomConfigs = () => { + return useFeature(Feature.AI_CUSTOM_CONFIGS) +} + // QUOTAS export const setAutomationLogsQuota = (value: number) => { diff --git a/packages/builder/src/components/backend/DataTable/modals/CreateEditColumn.svelte b/packages/builder/src/components/backend/DataTable/modals/CreateEditColumn.svelte index ffd064fc94..143e292c55 100644 --- a/packages/builder/src/components/backend/DataTable/modals/CreateEditColumn.svelte +++ b/packages/builder/src/components/backend/DataTable/modals/CreateEditColumn.svelte @@ -26,6 +26,7 @@ import { createEventDispatcher, getContext, onMount } from "svelte" import { cloneDeep } from "lodash/fp" import { tables, datasources } from "stores/builder" + import { licensing } from "stores/portal" import { TableNames, UNEDITABLE_USER_FIELDS } from "constants" import { FIELDS, @@ -35,6 +36,7 @@ } from "constants/backend" import { getAutoColumnInformation, buildAutoColumn } from "helpers/utils" import ConfirmDialog from "components/common/ConfirmDialog.svelte" + import AIFieldConfiguration from "components/common/AIFieldConfiguration.svelte" import ModalBindableInput from "components/common/bindings/ModalBindableInput.svelte" import { getBindings } from "components/backend/DataTable/formula" import JSONSchemaModal from "./JSONSchemaModal.svelte" @@ -99,6 +101,8 @@ let optionsValid = true $: rowGoldenSample = RowUtils.generateGoldenSample($rows) + $: aiEnabled = + $licensing.customAIConfigsEnabled || $licensing.budibaseAIEnabled $: if (primaryDisplay) { editableColumn.constraints.presence = { allowEmpty: false } } @@ -447,6 +451,7 @@ FIELDS.BOOLEAN, FIELDS.DATETIME, FIELDS.LINK, + ...(aiEnabled ? [FIELDS.AI] : []), FIELDS.LONGFORM, FIELDS.USER, FIELDS.USERS, @@ -784,6 +789,13 @@ /> + {:else if editableColumn.type === FieldType.AI} + {:else if editableColumn.type === FieldType.JSON}