diff --git a/.github/workflows/deploy-featurebranch.yml b/.github/workflows/deploy-featurebranch.yml index 0e19f0649f..872faa98fa 100644 --- a/.github/workflows/deploy-featurebranch.yml +++ b/.github/workflows/deploy-featurebranch.yml @@ -2,13 +2,11 @@ name: deploy-featurebranch on: pull_request: - types: [ - labeled, - # default types below (https://docs.github.com/en/actions/using-workflows/events-that-trigger-workflows#pull_request) - opened, - synchronize, - reopened, - ] + types: + - labeled + - opened + - synchronize + - reopened jobs: release: @@ -22,31 +20,21 @@ jobs: contains(github.event.pull_request.labels.*.name, 'feature-branch-enterprise') ) runs-on: ubuntu-latest + env: + PAYLOAD_BRANCH: ${{ github.head_ref }} + PAYLOAD_PR_NUMBER: ${{ github.event.pull_request.number }} + PAYLOAD_LICENSE_TYPE: | + ${{ + contains(github.event.pull_request.labels.*.name, 'feature-branch') && 'free' || + contains(github.event.pull_request.labels.*.name, 'feature-branch-pro') && 'pro' || + contains(github.event.pull_request.labels.*.name, 'feature-branch-team') && 'team' || + contains(github.event.pull_request.labels.*.name, 'feature-branch-business') && 'business' || + contains(github.event.pull_request.labels.*.name, 'feature-branch-enterprise') && 'enterprise' || 'free' + }} steps: - uses: actions/checkout@v4 - - name: Set PAYLOAD_LICENSE_TYPE - id: set_license_type - run: | - if [[ "${{ contains(github.event.pull_request.labels.*.name, 'feature-branch') }}" == "true" ]]; then - echo "PAYLOAD_LICENSE_TYPE=free" >> $GITHUB_ENV - elif [[ "${{ contains(github.event.pull_request.labels.*.name, 'feature-branch-pro') }}" == "true" ]]; then - echo "PAYLOAD_LICENSE_TYPE=pro" >> $GITHUB_ENV - elif [[ "${{ contains(github.event.pull_request.labels.*.name, 'feature-branch-team') }}" == "true" ]]; then - echo "PAYLOAD_LICENSE_TYPE=team" >> $GITHUB_ENV - elif [[ "${{ contains(github.event.pull_request.labels.*.name, 'feature-branch-business') }}" == "true" ]]; then - echo "PAYLOAD_LICENSE_TYPE=business" >> $GITHUB_ENV - elif [[ "${{ contains(github.event.pull_request.labels.*.name, 'feature-branch-enterprise') }}" == "true" ]]; then - echo "PAYLOAD_LICENSE_TYPE=enterprise" >> $GITHUB_ENV - else - echo "PAYLOAD_LICENSE_TYPE=free" >> $GITHUB_ENV - fi - - uses: passeidireto/trigger-external-workflow-action@main - env: - PAYLOAD_BRANCH: ${{ github.head_ref }} - PAYLOAD_PR_NUMBER: ${{ github.event.pull_request.number }} - PAYLOAD_LICENSE_TYPE: ${{ env.PAYLOAD_LICENSE_TYPE }} with: repository: budibase/budibase-deploys event: featurebranch-qa-deploy diff --git a/charts/budibase/templates/app-service-deployment.yaml b/charts/budibase/templates/app-service-deployment.yaml index 4d0560312f..278bd1767f 100644 --- a/charts/budibase/templates/app-service-deployment.yaml +++ b/charts/budibase/templates/app-service-deployment.yaml @@ -42,14 +42,12 @@ spec: {{ else }} value: http://{{ .Release.Name }}-svc-couchdb:{{ .Values.services.couchdb.port }} {{ end }} - {{ if .Values.globals.sqs.enabled }} - name: COUCH_DB_SQL_URL - {{ if .Values.globals.sqs.url }} - value: {{ .Values.globals.sqs.url }} - {{ else }} + {{ if .Values.globals.sqs.url }} + value: {{ .Values.globals.sqs.url | quote }} + {{ else }} value: http://{{ .Release.Name }}-svc-couchdb:{{ .Values.globals.sqs.port }} - {{ end }} - {{ end }} + {{ end }} {{ if .Values.services.couchdb.enabled }} - name: COUCH_DB_USER valueFrom: diff --git a/charts/budibase/templates/automation-worker-service-deployment.yaml b/charts/budibase/templates/automation-worker-service-deployment.yaml index 71089bd7ee..e0d43d0ce6 100644 --- a/charts/budibase/templates/automation-worker-service-deployment.yaml +++ b/charts/budibase/templates/automation-worker-service-deployment.yaml @@ -43,6 +43,12 @@ spec: {{ else }} value: http://{{ .Release.Name }}-svc-couchdb:{{ .Values.services.couchdb.port }} {{ end }} + - name: COUCH_DB_SQL_URL + {{ if .Values.globals.sqs.url }} + value: {{ .Values.globals.sqs.url | quote }} + {{ else }} + value: http://{{ .Release.Name }}-svc-couchdb:{{ .Values.globals.sqs.port }} + {{ end }} {{ if .Values.services.couchdb.enabled }} - name: COUCH_DB_USER valueFrom: diff --git a/charts/budibase/templates/worker-service-deployment.yaml b/charts/budibase/templates/worker-service-deployment.yaml index dcab33fa58..94fdd0b94e 100644 --- a/charts/budibase/templates/worker-service-deployment.yaml +++ b/charts/budibase/templates/worker-service-deployment.yaml @@ -56,14 +56,12 @@ spec: {{ else }} value: http://{{ .Release.Name }}-svc-couchdb:{{ .Values.services.couchdb.port }} {{ end }} - {{ if .Values.globals.sqs.enabled }} - name: COUCH_DB_SQL_URL - {{ if .Values.globals.sqs.url }} - value: {{ .Values.globals.sqs.url }} - {{ else }} + {{ if .Values.globals.sqs.url }} + value: {{ .Values.globals.sqs.url | quote }} + {{ else }} value: http://{{ .Release.Name }}-svc-couchdb:{{ .Values.globals.sqs.port }} - {{ end }} - {{ end }} + {{ end }} - name: API_ENCRYPTION_KEY valueFrom: secretKeyRef: diff --git a/charts/budibase/values.yaml b/charts/budibase/values.yaml index 2c1525bd90..de2cdb9474 100644 --- a/charts/budibase/values.yaml +++ b/charts/budibase/values.yaml @@ -139,9 +139,6 @@ globals: password: "" sqs: - # -- Whether to use the CouchDB "structured query service" or not. This is disabled by - # default for now, but will become the default in a future release. - enabled: false # @ignore url: "" # @ignore diff --git a/hosting/docker-compose.yaml b/hosting/docker-compose.yaml index a72b36aef1..c7a22eb2b3 100644 --- a/hosting/docker-compose.yaml +++ b/hosting/docker-compose.yaml @@ -5,7 +5,7 @@ version: "3" services: app-service: restart: unless-stopped - image: budibase.docker.scarf.sh/budibase/apps + image: budibase/apps container_name: bbapps environment: SELF_HOSTED: 1 @@ -35,7 +35,7 @@ services: worker-service: restart: unless-stopped - image: budibase.docker.scarf.sh/budibase/worker + image: budibase/worker container_name: bbworker environment: SELF_HOSTED: 1 @@ -97,7 +97,7 @@ services: couchdb-service: restart: unless-stopped - image: budibase/couchdb + image: budibase/couchdb:v3.3.3-sqs-v2.1.1 environment: - COUCHDB_PASSWORD=${COUCH_DB_PASSWORD} - COUCHDB_USER=${COUCH_DB_USER} diff --git a/hosting/single/Dockerfile b/hosting/single/Dockerfile index ded0bc17dc..a1230f3c37 100644 --- a/hosting/single/Dockerfile +++ b/hosting/single/Dockerfile @@ -69,6 +69,9 @@ WORKDIR /minio COPY scripts/install-minio.sh ./install.sh RUN chmod +x install.sh && ./install.sh +# setup redis +COPY hosting/single/redis.conf /etc/redis/redis.conf + # setup runner file WORKDIR / COPY hosting/single/runner.sh . diff --git a/hosting/single/redis.conf b/hosting/single/redis.conf new file mode 100644 index 0000000000..00740ffece --- /dev/null +++ b/hosting/single/redis.conf @@ -0,0 +1,7 @@ +dir "DATA_DIR/redis" + +appendonly yes +appendfsync everysec + +auto-aof-rewrite-percentage 100 +auto-aof-rewrite-min-size 64mb \ No newline at end of file diff --git a/hosting/single/runner.sh b/hosting/single/runner.sh index 95464dd031..d9b8719f0f 100644 --- a/hosting/single/runner.sh +++ b/hosting/single/runner.sh @@ -75,13 +75,17 @@ fi for LINE in $(cat ${DATA_DIR}/.env); do export $LINE; done ln -s ${DATA_DIR}/.env /app/.env ln -s ${DATA_DIR}/.env /worker/.env + # make these directories in runner, incase of mount mkdir -p ${DATA_DIR}/minio +mkdir -p ${DATA_DIR}/redis chown -R couchdb:couchdb ${DATA_DIR}/couch + +sed -i "s#DATA_DIR#${DATA_DIR}#g" /etc/redis/redis.conf if [[ -n "${REDIS_PASSWORD}" ]]; then - redis-server --requirepass $REDIS_PASSWORD > /dev/stdout 2>&1 & + redis-server /etc/redis/redis.conf --requirepass $REDIS_PASSWORD > /dev/stdout 2>&1 & else - redis-server > /dev/stdout 2>&1 & + redis-server /etc/redis/redis.conf > /dev/stdout 2>&1 & fi /bbcouch-runner.sh & diff --git a/lerna.json b/lerna.json index 13530e9aee..df50828b30 100644 --- a/lerna.json +++ b/lerna.json @@ -1,6 +1,6 @@ { "$schema": "node_modules/lerna/schemas/lerna-schema.json", - "version": "2.32.17", + "version": "2.33.2", "npmClient": "yarn", "packages": [ "packages/*", diff --git a/packages/backend-core/src/constants/misc.ts b/packages/backend-core/src/constants/misc.ts index aee099e10a..e2fd975e40 100644 --- a/packages/backend-core/src/constants/misc.ts +++ b/packages/backend-core/src/constants/misc.ts @@ -28,6 +28,7 @@ export enum Config { OIDC = "oidc", OIDC_LOGOS = "logos_oidc", SCIM = "scim", + AI = "AI", } export const MIN_VALID_DATE = new Date(-2147483647000) diff --git a/packages/backend-core/src/environment.ts b/packages/backend-core/src/environment.ts index 2ab8c550cc..4e93e8d9ee 100644 --- a/packages/backend-core/src/environment.ts +++ b/packages/backend-core/src/environment.ts @@ -54,30 +54,46 @@ function getPackageJsonFields(): { VERSION: string SERVICE_NAME: string } { - function findFileInAncestors( - fileName: string, - currentDir: string - ): string | null { - const filePath = `${currentDir}/${fileName}` - if (existsSync(filePath)) { - return filePath + function getParentFile(file: string) { + function findFileInAncestors( + fileName: string, + currentDir: string + ): string | null { + const filePath = `${currentDir}/${fileName}` + if (existsSync(filePath)) { + return filePath + } + + const parentDir = `${currentDir}/..` + if (parentDir === currentDir) { + // reached root directory + return null + } + + return findFileInAncestors(fileName, parentDir) } - const parentDir = `${currentDir}/..` - if (parentDir === currentDir) { - // reached root directory - return null - } + const packageJsonFile = findFileInAncestors(file, process.cwd()) + const content = readFileSync(packageJsonFile!, "utf-8") + const parsedContent = JSON.parse(content) + return parsedContent + } - return findFileInAncestors(fileName, parentDir) + let localVersion: string | undefined + if (isDev() && !isTest()) { + try { + const lerna = getParentFile("lerna.json") + localVersion = lerna.version + } catch { + // + } } try { - const packageJsonFile = findFileInAncestors("package.json", process.cwd()) - const content = readFileSync(packageJsonFile!, "utf-8") - const parsedContent = JSON.parse(content) + const parsedContent = getParentFile("package.json") return { - VERSION: process.env.BUDIBASE_VERSION || parsedContent.version, + VERSION: + localVersion || process.env.BUDIBASE_VERSION || parsedContent.version, SERVICE_NAME: parsedContent.name, } } catch { diff --git a/packages/backend-core/src/features/features.ts b/packages/backend-core/src/features/features.ts index 20b207bb02..e95472a784 100644 --- a/packages/backend-core/src/features/features.ts +++ b/packages/backend-core/src/features/features.ts @@ -269,7 +269,7 @@ export class FlagSet, T extends { [key: string]: V }> { export const flags = new FlagSet({ DEFAULT_VALUES: Flag.boolean(env.isDev()), AUTOMATION_BRANCHING: Flag.boolean(env.isDev()), - SQS: Flag.boolean(env.isDev()), + SQS: Flag.boolean(true), [FeatureFlag.AI_CUSTOM_CONFIGS]: Flag.boolean(env.isDev()), [FeatureFlag.ENRICHED_RELATIONSHIPS]: Flag.boolean(env.isDev()), [FeatureFlag.TABLES_DEFAULT_ADMIN]: Flag.boolean(env.isDev()), diff --git a/packages/backend-core/src/features/tests/features.spec.ts b/packages/backend-core/src/features/tests/features.spec.ts index 9af8a8f4bb..ced874f4af 100644 --- a/packages/backend-core/src/features/tests/features.spec.ts +++ b/packages/backend-core/src/features/tests/features.spec.ts @@ -10,6 +10,7 @@ const schema = { TEST_BOOLEAN: Flag.boolean(false), TEST_STRING: Flag.string("default value"), TEST_NUMBER: Flag.number(0), + TEST_BOOLEAN_DEFAULT_TRUE: Flag.boolean(true), } const flags = new FlagSet(schema) @@ -123,6 +124,11 @@ describe("feature flags", () => { }, expected: flags.defaults(), }, + { + it: "should be possible to override a default true flag to false", + environmentFlags: "default:!TEST_BOOLEAN_DEFAULT_TRUE", + expected: { TEST_BOOLEAN_DEFAULT_TRUE: false }, + }, ])( "$it", async ({ diff --git a/packages/backend-core/src/security/roles.ts b/packages/backend-core/src/security/roles.ts index fa2d114d7d..fad5f7cb74 100644 --- a/packages/backend-core/src/security/roles.ts +++ b/packages/backend-core/src/security/roles.ts @@ -1,3 +1,4 @@ +import semver from "semver" import { BuiltinPermissionID, PermissionLevel } from "./permissions" import { prefixRoleID, @@ -7,7 +8,13 @@ import { doWithDB, } from "../db" import { getAppDB } from "../context" -import { Screen, Role as RoleDoc, RoleUIMetadata } from "@budibase/types" +import { + Screen, + Role as RoleDoc, + RoleUIMetadata, + Database, + App, +} from "@budibase/types" import cloneDeep from "lodash/fp/cloneDeep" import { RoleColor } from "@budibase/shared-core" @@ -23,14 +30,6 @@ const BUILTIN_IDS = { BUILDER: "BUILDER", } -// exclude internal roles like builder -const EXTERNAL_BUILTIN_ROLE_IDS = [ - BUILTIN_IDS.ADMIN, - BUILTIN_IDS.POWER, - BUILTIN_IDS.BASIC, - BUILTIN_IDS.PUBLIC, -] - export const RoleIDVersion = { // original version, with a UUID based ID UUID: undefined, @@ -319,7 +318,7 @@ export async function getAllRoles(appId?: string): Promise { } return internal(appDB) } - async function internal(db: any) { + async function internal(db: Database | undefined) { let roles: RoleDoc[] = [] if (db) { const body = await db.allDocs( @@ -334,8 +333,26 @@ export async function getAllRoles(appId?: string): Promise { } const builtinRoles = getBuiltinRoles() + // exclude internal roles like builder + let externalBuiltinRoles = [] + + if (!db || (await shouldIncludePowerRole(db))) { + externalBuiltinRoles = [ + BUILTIN_IDS.ADMIN, + BUILTIN_IDS.POWER, + BUILTIN_IDS.BASIC, + BUILTIN_IDS.PUBLIC, + ] + } else { + externalBuiltinRoles = [ + BUILTIN_IDS.ADMIN, + BUILTIN_IDS.BASIC, + BUILTIN_IDS.PUBLIC, + ] + } + // need to combine builtin with any DB record of them (for sake of permissions) - for (let builtinRoleId of EXTERNAL_BUILTIN_ROLE_IDS) { + for (let builtinRoleId of externalBuiltinRoles) { const builtinRole = builtinRoles[builtinRoleId] const dbBuiltin = roles.filter( dbRole => @@ -366,6 +383,18 @@ export async function getAllRoles(appId?: string): Promise { } } +async function shouldIncludePowerRole(db: Database) { + const app = await db.tryGet(DocumentType.APP_METADATA) + const creationVersion = app?.creationVersion + if (!creationVersion || !semver.valid(creationVersion)) { + // Old apps don't have creationVersion, so we should include it for backward compatibility + return true + } + + const isGreaterThan3x = semver.gte(creationVersion, "3.0.0") + return !isGreaterThan3x +} + export class AccessController { userHierarchies: { [key: string]: string[] } constructor() { diff --git a/packages/backend-core/src/sql/sql.ts b/packages/backend-core/src/sql/sql.ts index 382eca3f76..2b75752aec 100644 --- a/packages/backend-core/src/sql/sql.ts +++ b/packages/backend-core/src/sql/sql.ts @@ -23,12 +23,14 @@ import { InternalSearchFilterOperator, JsonFieldMetadata, JsonTypes, + LogicalOperator, Operation, prefixed, QueryJson, QueryOptions, RangeOperator, RelationshipsJson, + SearchFilterKey, SearchFilters, SortOrder, SqlClient, @@ -96,6 +98,22 @@ function isSqs(table: Table): boolean { ) } +const allowEmptyRelationships: Record = { + [BasicOperator.EQUAL]: false, + [BasicOperator.NOT_EQUAL]: true, + [BasicOperator.EMPTY]: false, + [BasicOperator.NOT_EMPTY]: true, + [BasicOperator.FUZZY]: false, + [BasicOperator.STRING]: false, + [RangeOperator.RANGE]: false, + [ArrayOperator.CONTAINS]: false, + [ArrayOperator.NOT_CONTAINS]: true, + [ArrayOperator.CONTAINS_ANY]: false, + [ArrayOperator.ONE_OF]: false, + [LogicalOperator.AND]: false, + [LogicalOperator.OR]: false, +} + class InternalBuilder { private readonly client: SqlClient private readonly query: QueryJson @@ -405,31 +423,48 @@ class InternalBuilder { addRelationshipForFilter( query: Knex.QueryBuilder, + allowEmptyRelationships: boolean, filterKey: string, - whereCb: (query: Knex.QueryBuilder) => Knex.QueryBuilder + whereCb: (filterKey: string, query: Knex.QueryBuilder) => Knex.QueryBuilder ): Knex.QueryBuilder { const mainKnex = this.knex const { relationships, endpoint, tableAliases: aliases } = this.query const tableName = endpoint.entityId const fromAlias = aliases?.[tableName] || tableName - const matches = (possibleTable: string) => - filterKey.startsWith(`${possibleTable}`) + const matches = (value: string) => + filterKey.match(new RegExp(`^${value}\\.`)) if (!relationships) { return query } for (const relationship of relationships) { const relatedTableName = relationship.tableName const toAlias = aliases?.[relatedTableName] || relatedTableName + + const matchesTableName = matches(relatedTableName) || matches(toAlias) + const matchesRelationName = matches(relationship.column) + // this is the relationship which is being filtered if ( - (matches(relatedTableName) || matches(toAlias)) && + (matchesTableName || matchesRelationName) && relationship.to && relationship.tableName ) { - let subQuery = mainKnex + const joinTable = mainKnex .select(mainKnex.raw(1)) .from({ [toAlias]: relatedTableName }) + let subQuery = joinTable.clone() const manyToMany = validateManyToMany(relationship) + let updatedKey + + if (!matchesTableName) { + updatedKey = filterKey.replace( + new RegExp(`^${relationship.column}.`), + `${aliases![relationship.tableName]}.` + ) + } else { + updatedKey = filterKey + } + if (manyToMany) { const throughAlias = aliases?.[manyToMany.through] || relationship.through @@ -440,7 +475,6 @@ class InternalBuilder { subQuery = subQuery // add a join through the junction table .innerJoin(throughTable, function () { - // @ts-ignore this.on( `${toAlias}.${manyToMany.toPrimary}`, "=", @@ -460,18 +494,38 @@ class InternalBuilder { if (this.client === SqlClient.SQL_LITE) { subQuery = this.addJoinFieldCheck(subQuery, manyToMany) } + + query = query.where(q => { + q.whereExists(whereCb(updatedKey, subQuery)) + if (allowEmptyRelationships) { + q.orWhereNotExists( + joinTable.clone().innerJoin(throughTable, function () { + this.on( + `${fromAlias}.${manyToMany.fromPrimary}`, + "=", + `${throughAlias}.${manyToMany.from}` + ) + }) + ) + } + }) } else { + const toKey = `${toAlias}.${relationship.to}` + const foreignKey = `${fromAlias}.${relationship.from}` // "join" to the main table, making sure the ID matches that of the main subQuery = subQuery.where( - `${toAlias}.${relationship.to}`, + toKey, "=", - mainKnex.raw( - this.quotedIdentifier(`${fromAlias}.${relationship.from}`) - ) + mainKnex.raw(this.quotedIdentifier(foreignKey)) ) + + query = query.where(q => { + q.whereExists(whereCb(updatedKey, subQuery.clone())) + if (allowEmptyRelationships) { + q.orWhereNotExists(subQuery) + } + }) } - query = query.whereExists(whereCb(subQuery)) - break } } return query @@ -502,6 +556,7 @@ class InternalBuilder { } function iterate( structure: AnySearchFilter, + operation: SearchFilterKey, fn: ( query: Knex.QueryBuilder, key: string, @@ -521,8 +576,11 @@ class InternalBuilder { const [filterTableName, ...otherProperties] = key.split(".") const property = otherProperties.join(".") const alias = getTableAlias(filterTableName) - return fn(q, alias ? `${alias}.${property}` : property, value) + return q.andWhere(subquery => + fn(subquery, alias ? `${alias}.${property}` : property, value) + ) } + for (const key in structure) { const value = structure[key] const updatedKey = dbCore.removeKeyNumbering(key) @@ -552,9 +610,17 @@ class InternalBuilder { value ) } else if (shouldProcessRelationship) { - query = builder.addRelationshipForFilter(query, updatedKey, q => { - return handleRelationship(q, updatedKey, value) - }) + if (allOr) { + query = query.or + } + query = builder.addRelationshipForFilter( + query, + allowEmptyRelationships[operation], + updatedKey, + (updatedKey, q) => { + return handleRelationship(q, updatedKey, value) + } + ) } } } @@ -586,7 +652,7 @@ class InternalBuilder { return `[${value.join(",")}]` } if (this.client === SqlClient.POSTGRES) { - iterate(mode, (q, key, value) => { + iterate(mode, ArrayOperator.CONTAINS, (q, key, value) => { const wrap = any ? "" : "'" const op = any ? "\\?| array" : "@>" const fieldNames = key.split(/\./g) @@ -604,7 +670,7 @@ class InternalBuilder { this.client === SqlClient.MARIADB ) { const jsonFnc = any ? "JSON_OVERLAPS" : "JSON_CONTAINS" - iterate(mode, (q, key, value) => { + iterate(mode, ArrayOperator.CONTAINS, (q, key, value) => { return q[rawFnc]( `${not}COALESCE(${jsonFnc}(${key}, '${stringifyArray( value @@ -613,7 +679,7 @@ class InternalBuilder { }) } else { const andOr = mode === filters?.containsAny ? " OR " : " AND " - iterate(mode, (q, key, value) => { + iterate(mode, ArrayOperator.CONTAINS, (q, key, value) => { let statement = "" const identifier = this.quotedIdentifier(key) for (let i in value) { @@ -667,6 +733,7 @@ class InternalBuilder { const fnc = allOr ? "orWhereIn" : "whereIn" iterate( filters.oneOf, + ArrayOperator.ONE_OF, (q, key: string, array) => { if (this.client === SqlClient.ORACLE) { key = this.convertClobs(key) @@ -691,7 +758,7 @@ class InternalBuilder { ) } if (filters.string) { - iterate(filters.string, (q, key, value) => { + iterate(filters.string, BasicOperator.STRING, (q, key, value) => { const fnc = allOr ? "orWhere" : "where" // postgres supports ilike, nothing else does if (this.client === SqlClient.POSTGRES) { @@ -706,10 +773,10 @@ class InternalBuilder { }) } if (filters.fuzzy) { - iterate(filters.fuzzy, like) + iterate(filters.fuzzy, BasicOperator.FUZZY, like) } if (filters.range) { - iterate(filters.range, (q, key, value) => { + iterate(filters.range, RangeOperator.RANGE, (q, key, value) => { const isEmptyObject = (val: any) => { return ( val && @@ -775,7 +842,7 @@ class InternalBuilder { }) } if (filters.equal) { - iterate(filters.equal, (q, key, value) => { + iterate(filters.equal, BasicOperator.EQUAL, (q, key, value) => { const fnc = allOr ? "orWhereRaw" : "whereRaw" if (this.client === SqlClient.MS_SQL) { return q[fnc]( @@ -795,7 +862,7 @@ class InternalBuilder { }) } if (filters.notEqual) { - iterate(filters.notEqual, (q, key, value) => { + iterate(filters.notEqual, BasicOperator.NOT_EQUAL, (q, key, value) => { const fnc = allOr ? "orWhereRaw" : "whereRaw" if (this.client === SqlClient.MS_SQL) { return q[fnc]( @@ -816,13 +883,13 @@ class InternalBuilder { }) } if (filters.empty) { - iterate(filters.empty, (q, key) => { + iterate(filters.empty, BasicOperator.EMPTY, (q, key) => { const fnc = allOr ? "orWhereNull" : "whereNull" return q[fnc](key) }) } if (filters.notEmpty) { - iterate(filters.notEmpty, (q, key) => { + iterate(filters.notEmpty, BasicOperator.NOT_EMPTY, (q, key) => { const fnc = allOr ? "orWhereNotNull" : "whereNotNull" return q[fnc](key) }) @@ -1218,12 +1285,10 @@ class InternalBuilder { }) : undefined if (!throughTable) { - // @ts-ignore query = query.leftJoin(toTableWithSchema, function () { for (let relationship of columns) { const from = relationship.from, to = relationship.to - // @ts-ignore this.orOn(`${fromAlias}.${from}`, "=", `${toAlias}.${to}`) } }) @@ -1234,7 +1299,6 @@ class InternalBuilder { for (let relationship of columns) { const fromPrimary = relationship.fromPrimary const from = relationship.from - // @ts-ignore this.orOn( `${fromAlias}.${fromPrimary}`, "=", @@ -1246,7 +1310,6 @@ class InternalBuilder { for (let relationship of columns) { const toPrimary = relationship.toPrimary const to = relationship.to - // @ts-ignore this.orOn(`${toAlias}.${toPrimary}`, `${throughAlias}.${to}`) } }) @@ -1288,7 +1351,8 @@ class InternalBuilder { schema.constraints?.presence === true || schema.type === FieldType.FORMULA || schema.type === FieldType.AUTO || - schema.type === FieldType.LINK + schema.type === FieldType.LINK || + schema.type === FieldType.AI ) { continue } diff --git a/packages/backend-core/src/sql/sqlTable.ts b/packages/backend-core/src/sql/sqlTable.ts index f5b02cc4e4..84f4e290aa 100644 --- a/packages/backend-core/src/sql/sqlTable.ts +++ b/packages/backend-core/src/sql/sqlTable.ts @@ -17,7 +17,7 @@ import SchemaBuilder = Knex.SchemaBuilder import CreateTableBuilder = Knex.CreateTableBuilder function isIgnoredType(type: FieldType) { - const ignored = [FieldType.LINK, FieldType.FORMULA] + const ignored = [FieldType.LINK, FieldType.FORMULA, FieldType.AI] return ignored.indexOf(type) !== -1 } @@ -144,6 +144,9 @@ function generateSchema( case FieldType.FORMULA: // This is allowed, but nothing to do on the external datasource break + case FieldType.AI: + // This is allowed, but nothing to do on the external datasource + break case FieldType.ATTACHMENTS: case FieldType.ATTACHMENT_SINGLE: case FieldType.SIGNATURE_SINGLE: diff --git a/packages/backend-core/tests/core/utilities/mocks/licenses.ts b/packages/backend-core/tests/core/utilities/mocks/licenses.ts index bc9a3b635c..5ba6fb36a1 100644 --- a/packages/backend-core/tests/core/utilities/mocks/licenses.ts +++ b/packages/backend-core/tests/core/utilities/mocks/licenses.ts @@ -102,6 +102,14 @@ export const useAppBuilders = () => { return useFeature(Feature.APP_BUILDERS) } +export const useBudibaseAI = () => { + return useFeature(Feature.BUDIBASE_AI) +} + +export const useAICustomConfigs = () => { + return useFeature(Feature.AI_CUSTOM_CONFIGS) +} + // QUOTAS export const setAutomationLogsQuota = (value: number) => { diff --git a/packages/builder/src/components/backend/DataTable/modals/CreateEditColumn.svelte b/packages/builder/src/components/backend/DataTable/modals/CreateEditColumn.svelte index 19cc4ec1c0..143e292c55 100644 --- a/packages/builder/src/components/backend/DataTable/modals/CreateEditColumn.svelte +++ b/packages/builder/src/components/backend/DataTable/modals/CreateEditColumn.svelte @@ -26,6 +26,7 @@ import { createEventDispatcher, getContext, onMount } from "svelte" import { cloneDeep } from "lodash/fp" import { tables, datasources } from "stores/builder" + import { licensing } from "stores/portal" import { TableNames, UNEDITABLE_USER_FIELDS } from "constants" import { FIELDS, @@ -35,6 +36,7 @@ } from "constants/backend" import { getAutoColumnInformation, buildAutoColumn } from "helpers/utils" import ConfirmDialog from "components/common/ConfirmDialog.svelte" + import AIFieldConfiguration from "components/common/AIFieldConfiguration.svelte" import ModalBindableInput from "components/common/bindings/ModalBindableInput.svelte" import { getBindings } from "components/backend/DataTable/formula" import JSONSchemaModal from "./JSONSchemaModal.svelte" @@ -50,18 +52,13 @@ import { isEnabled } from "helpers/featureFlags" import { getUserBindings } from "dataBinding" - const AUTO_TYPE = FieldType.AUTO - const FORMULA_TYPE = FieldType.FORMULA - const LINK_TYPE = FieldType.LINK - const STRING_TYPE = FieldType.STRING - const NUMBER_TYPE = FieldType.NUMBER - const JSON_TYPE = FieldType.JSON - const DATE_TYPE = FieldType.DATETIME + export let field const dispatch = createEventDispatcher() const { dispatch: gridDispatch, rows } = getContext("grid") - - export let field + const SafeID = `${makePropSafe("user")}.${makePropSafe("_id")}` + const SingleUserDefault = `{{ ${SafeID} }}` + const MultiUserDefault = `{{ js "${btoa(`return [$("${SafeID}")]`)}" }}` let mounted = false let originalName @@ -104,13 +101,15 @@ let optionsValid = true $: rowGoldenSample = RowUtils.generateGoldenSample($rows) + $: aiEnabled = + $licensing.customAIConfigsEnabled || $licensing.budibaseAIEnabled $: if (primaryDisplay) { editableColumn.constraints.presence = { allowEmpty: false } } $: { // this parses any changes the user has made when creating a new internal relationship // into what we expect the schema to look like - if (editableColumn.type === LINK_TYPE) { + if (editableColumn.type === FieldType.LINK) { relationshipTableIdPrimary = table._id if (relationshipPart1 === PrettyRelationshipDefinitions.ONE) { relationshipOpts2 = relationshipOpts2.filter( @@ -147,7 +146,7 @@ UNEDITABLE_USER_FIELDS.includes(editableColumn.name) $: invalid = !editableColumn?.name || - (editableColumn?.type === LINK_TYPE && !editableColumn?.tableId) || + (editableColumn?.type === FieldType.LINK && !editableColumn?.tableId) || Object.keys(errors).length !== 0 || !optionsValid $: errors = checkErrors(editableColumn) @@ -173,9 +172,9 @@ $: defaultValuesEnabled = isEnabled("DEFAULT_VALUES") $: canHaveDefault = !required && canHaveDefaultColumn(editableColumn.type) $: canBeRequired = - editableColumn?.type !== LINK_TYPE && + editableColumn?.type !== FieldType.LINK && !uneditable && - editableColumn?.type !== AUTO_TYPE && + editableColumn?.type !== FieldType.AUTO && !editableColumn.autocolumn $: hasDefault = editableColumn?.default != null && editableColumn?.default !== "" @@ -224,7 +223,7 @@ function makeFieldId(type, subtype, autocolumn) { // don't make field IDs for auto types - if (type === AUTO_TYPE || autocolumn) { + if (type === FieldType.AUTO || autocolumn) { return type.toUpperCase() } else if ( type === FieldType.BB_REFERENCE || @@ -249,7 +248,7 @@ // Here we are setting the relationship values based on the editableColumn // This part of the code is used when viewing an existing field hence the check // for the tableId - if (editableColumn.type === LINK_TYPE && editableColumn.tableId) { + if (editableColumn.type === FieldType.LINK && editableColumn.tableId) { relationshipTableIdPrimary = table._id relationshipTableIdSecondary = editableColumn.tableId if (editableColumn.relationshipType in relationshipMap) { @@ -290,14 +289,14 @@ delete saveColumn.fieldId - if (saveColumn.type === AUTO_TYPE) { + if (saveColumn.type === FieldType.AUTO) { saveColumn = buildAutoColumn( $tables.selected.name, saveColumn.name, saveColumn.subtype ) } - if (saveColumn.type !== LINK_TYPE) { + if (saveColumn.type !== FieldType.LINK) { delete saveColumn.fieldName } @@ -384,9 +383,9 @@ editableColumn.subtype = definition.subtype // Default relationships many to many - if (editableColumn.type === LINK_TYPE) { + if (editableColumn.type === FieldType.LINK) { editableColumn.relationshipType = RelationshipType.MANY_TO_MANY - } else if (editableColumn.type === FORMULA_TYPE) { + } else if (editableColumn.type === FieldType.FORMULA) { editableColumn.formulaType = "dynamic" } } @@ -452,6 +451,7 @@ FIELDS.BOOLEAN, FIELDS.DATETIME, FIELDS.LINK, + ...(aiEnabled ? [FIELDS.AI] : []), FIELDS.LONGFORM, FIELDS.USER, FIELDS.USERS, @@ -505,17 +505,23 @@ fieldToCheck.constraints = {} } // some string types may have been built by server, may not always have constraints - if (fieldToCheck.type === STRING_TYPE && !fieldToCheck.constraints.length) { + if ( + fieldToCheck.type === FieldType.STRING && + !fieldToCheck.constraints.length + ) { fieldToCheck.constraints.length = {} } // some number types made server-side will be missing constraints if ( - fieldToCheck.type === NUMBER_TYPE && + fieldToCheck.type === FieldType.NUMBER && !fieldToCheck.constraints.numericality ) { fieldToCheck.constraints.numericality = {} } - if (fieldToCheck.type === DATE_TYPE && !fieldToCheck.constraints.datetime) { + if ( + fieldToCheck.type === FieldType.DATETIME && + !fieldToCheck.constraints.datetime + ) { fieldToCheck.constraints.datetime = {} } } @@ -590,13 +596,13 @@ on:input={e => { if ( !uneditable && - !(linkEditDisabled && editableColumn.type === LINK_TYPE) + !(linkEditDisabled && editableColumn.type === FieldType.LINK) ) { editableColumn.name = e.target.value } }} disabled={uneditable || - (linkEditDisabled && editableColumn.type === LINK_TYPE)} + (linkEditDisabled && editableColumn.type === FieldType.LINK)} error={errors?.name} /> {/if} @@ -610,7 +616,7 @@ getOptionValue={field => field.fieldId} getOptionIcon={field => field.icon} isOptionEnabled={option => { - if (option.type === AUTO_TYPE) { + if (option.type === FieldType.AUTO) { return availableAutoColumnKeys?.length > 0 } return true @@ -653,7 +659,7 @@ bind:optionColors={editableColumn.optionColors} bind:valid={optionsValid} /> - {:else if editableColumn.type === DATE_TYPE && !editableColumn.autocolumn} + {:else if editableColumn.type === FieldType.DATETIME && !editableColumn.autocolumn}
@@ -740,7 +746,7 @@ {tableOptions} {errors} /> - {:else if editableColumn.type === FORMULA_TYPE} + {:else if editableColumn.type === FieldType.FORMULA} {#if !externalTable}
@@ -783,12 +789,19 @@ />
- {:else if editableColumn.type === JSON_TYPE} + {:else if editableColumn.type === FieldType.AI} + + {:else if editableColumn.type === FieldType.JSON} {/if} - {#if editableColumn.type === AUTO_TYPE || editableColumn.autocolumn} + {#if editableColumn.type === FieldType.AUTO || editableColumn.autocolumn} +{#if aiField.operation} + {#each Object.keys(OperationField) as key} + {#if OperationField[key] === OperationFieldTypes.BINDABLE_TEXT} + (aiField[key] = e.detail)} + value={aiField[key]} + {bindings} + allowJS + {context} + /> + {:else if OperationField[key] === OperationFieldTypes.MULTI_COLUMN} + + {:else if OperationField[key] === OperationFieldTypes.COLUMN} + { + let totalWaitMs = 0 + // eslint-disable-next-line no-constant-condition + while (true) { + const waitForMs = 5000 + Math.random() * 5000 + await new Promise(resolve => setTimeout(resolve, waitForMs)) + totalWaitMs += waitForMs + const isMigrated = await isMigrationDone() - - const timeoutMs = timeoutSeconds * 1000 - if (!isMigrated || secondsWaited <= minTimeSeconds) { - if (loadTime + timeoutMs > Date.now()) { - secondsWaited += 1 - return checkMigrationsFinished() - } - - return migrationTimeout() + if (isMigrated) { + onMigrationDone() + return } - onMigrationDone() - }, intervalMs) + if (totalWaitMs > timeoutSeconds * 1000) { + timedOut = true + return + } + } } checkMigrationsFinished() - - function migrationTimeout() { - timedOut = true - }
diff --git a/packages/frontend-core/src/components/grid/cells/AICell.svelte b/packages/frontend-core/src/components/grid/cells/AICell.svelte new file mode 100644 index 0000000000..38e81cefd3 --- /dev/null +++ b/packages/frontend-core/src/components/grid/cells/AICell.svelte @@ -0,0 +1,99 @@ + + + + +
+
+ {value || ""} +
+
+ +{#if isOpen} + +