From ab5f50d2b8db2be4be57cf54eb64d77473e9d2e8 Mon Sep 17 00:00:00 2001 From: mike12345567 Date: Fri, 23 Aug 2024 18:00:52 +0100 Subject: [PATCH 01/85] Implementing a JSON aggregate method of selecting relationships. --- packages/backend-core/src/sql/sql.ts | 130 +++++++++++++++++++-------- 1 file changed, 95 insertions(+), 35 deletions(-) diff --git a/packages/backend-core/src/sql/sql.ts b/packages/backend-core/src/sql/sql.ts index 6a8518ea7b..1c87abf5ae 100644 --- a/packages/backend-core/src/sql/sql.ts +++ b/packages/backend-core/src/sql/sql.ts @@ -126,12 +126,20 @@ class InternalBuilder { } private generateSelectStatement(): (string | Knex.Raw)[] | "*" { - const { resource, meta } = this.query + const { endpoint, resource, meta, tableAliases } = this.query if (!resource || !resource.fields || resource.fields.length === 0) { return "*" } + // no relationships - select everything in SQLite + if (this.client === SqlClient.SQL_LITE) { + const alias = tableAliases?.[endpoint.entityId] + ? tableAliases?.[endpoint.entityId] + : endpoint.entityId + return [this.knex.raw(`${this.quote(alias)}.*`)] + } + const schema = meta.table.schema return resource.fields.map(field => { const parts = field.split(/\./g) @@ -745,16 +753,83 @@ class InternalBuilder { return withSchema } + addJsonRelationships( + query: Knex.QueryBuilder, + fromTable: string, + relationships: RelationshipsJson[] + ): Knex.QueryBuilder { + const { resource, tableAliases: aliases, endpoint } = this.query + const fields = resource?.fields || [] + const jsonField = (field: string) => { + const unAliased = field.split(".").slice(1).join(".") + return `'${unAliased}',${field}` + } + for (let relationship of relationships) { + const { + tableName: toTable, + through: throughTable, + to: toKey, + from: fromKey, + fromPrimary, + toPrimary, + } = relationship + // skip invalid relationships + if (!toTable || !fromTable || !fromPrimary || !toPrimary) { + continue + } + if (!throughTable) { + throw new Error("Only many-to-many implemented for JSON relationships") + } + const toAlias = aliases?.[toTable] || toTable, + throughAlias = aliases?.[throughTable] || throughTable, + fromAlias = aliases?.[fromTable] || fromTable + let toTableWithSchema = this.tableNameWithSchema(toTable, { + alias: toAlias, + schema: endpoint.schema, + }) + let throughTableWithSchema = this.tableNameWithSchema(throughTable, { + alias: throughAlias, + schema: endpoint.schema, + }) + const relationshipFields = fields.filter( + field => field.split(".")[0] === toAlias + ) + const fieldList: string = relationshipFields + .map(field => jsonField(field)) + .join(",") + let rawJsonArray: Knex.Raw + switch (this.client) { + case SqlClient.SQL_LITE: + rawJsonArray = this.knex.raw( + `json_group_array(json_object(${fieldList}))` + ) + break + default: + throw new Error(`JSON relationships not implement for ${this.client}`) + } + const subQuery = this.knex + .select(rawJsonArray) + .from(toTableWithSchema) + .join(throughTableWithSchema, function () { + this.on(`${toAlias}.${toPrimary}`, "=", `${throughAlias}.${toKey}`) + }) + .where( + `${throughAlias}.${fromKey}`, + "=", + this.knex.raw(this.quotedIdentifier(`${fromAlias}.${fromPrimary}`)) + ) + query = query.select({ [relationship.column]: subQuery }) + } + return query + } + addRelationships( query: Knex.QueryBuilder, fromTable: string, - relationships: RelationshipsJson[] | undefined, - schema: string | undefined, + relationships: RelationshipsJson[], + schema?: string, aliases?: Record ): Knex.QueryBuilder { - if (!relationships) { - return query - } const tableSets: Record = {} // aggregate into table sets (all the same to tables) for (let relationship of relationships) { @@ -957,42 +1032,27 @@ class InternalBuilder { if (foundOffset != null) { query = query.offset(foundOffset) } - // add sorting to pre-query - // no point in sorting when counting - query = this.addSorting(query) } - // add filters to the query (where) - query = this.addFilters(query, filters) - const alias = tableAliases?.[tableName] || tableName - let preQuery: Knex.QueryBuilder = this.knex({ - // the typescript definition for the knex constructor doesn't support this - // syntax, but it is the only way to alias a pre-query result as part of - // a query - there is an alias dictionary type, but it assumes it can only - // be a table name, not a pre-query - [alias]: query as any, - }) // if counting, use distinct count, else select - preQuery = !counting - ? preQuery.select(this.generateSelectStatement()) - : this.addDistinctCount(preQuery) + query = !counting + ? query.select(this.generateSelectStatement()) + : this.addDistinctCount(query) // have to add after as well (this breaks MS-SQL) if (this.client !== SqlClient.MS_SQL && !counting) { - preQuery = this.addSorting(preQuery) + query = this.addSorting(query) } // handle joins - query = this.addRelationships( - preQuery, - tableName, - relationships, - endpoint.schema, - tableAliases - ) - - // add a base limit over the whole query - // if counting we can't set this limit - if (limits?.base) { - query = query.limit(limits.base) + if (relationships && this.client === SqlClient.SQL_LITE) { + query = this.addJsonRelationships(query, tableName, relationships) + } else if (relationships) { + query = this.addRelationships( + query, + tableName, + relationships, + endpoint.schema, + tableAliases + ) } return this.addFilters(query, filters, { relationship: true }) From 80f3e5954bf5ecb84a8559789fcc50f5b9f5e474 Mon Sep 17 00:00:00 2001 From: mike12345567 Date: Fri, 23 Aug 2024 18:30:29 +0100 Subject: [PATCH 02/85] Getting processing of SQS relationships working. --- .../src/api/controllers/row/utils/basic.ts | 11 +++++++--- .../src/api/controllers/row/utils/utils.ts | 20 ++++++++++--------- .../src/sdk/app/rows/search/internal/sqs.ts | 13 +++++++++++- 3 files changed, 31 insertions(+), 13 deletions(-) diff --git a/packages/server/src/api/controllers/row/utils/basic.ts b/packages/server/src/api/controllers/row/utils/basic.ts index f28f650422..8f3607bc73 100644 --- a/packages/server/src/api/controllers/row/utils/basic.ts +++ b/packages/server/src/api/controllers/row/utils/basic.ts @@ -1,5 +1,5 @@ // need to handle table name + field or just field, depending on if relationships used -import { FieldType, Row, Table } from "@budibase/types" +import { FieldSchema, FieldType, Row, Table } from "@budibase/types" import { helpers, PROTECTED_INTERNAL_COLUMNS } from "@budibase/shared-core" import { generateRowIdField } from "../../../../integrations/utils" @@ -82,7 +82,7 @@ export function basicProcessing({ value = value.toString() } // all responses include "select col as table.col" so that overlaps are handled - if (value != null) { + else if (value != null) { thisRow[fieldName] = value } } @@ -93,12 +93,17 @@ export function basicProcessing({ } else { const columns = Object.keys(table.schema) for (let internalColumn of [...PROTECTED_INTERNAL_COLUMNS, ...columns]) { - thisRow[internalColumn] = extractFieldValue({ + const schema: FieldSchema | undefined = table.schema[internalColumn] + let value = extractFieldValue({ row, tableName: table._id!, fieldName: internalColumn, isLinked, }) + if (sqs && schema?.type === FieldType.LINK && typeof value === "string") { + value = JSON.parse(value) + } + thisRow[internalColumn] = value } } return thisRow diff --git a/packages/server/src/api/controllers/row/utils/utils.ts b/packages/server/src/api/controllers/row/utils/utils.ts index 911cfe8d5b..fdf05baf3c 100644 --- a/packages/server/src/api/controllers/row/utils/utils.ts +++ b/packages/server/src/api/controllers/row/utils/utils.ts @@ -147,7 +147,7 @@ export async function sqlOutputProcessing( row._id = rowId } // this is a relationship of some sort - if (finalRows[rowId]) { + if (!opts?.sqs && finalRows[rowId]) { finalRows = await updateRelationshipColumns( table, tables, @@ -174,14 +174,16 @@ export async function sqlOutputProcessing( finalRows[thisRow._id] = fixBooleanFields({ row: thisRow, table }) // do this at end once its been added to the final rows - finalRows = await updateRelationshipColumns( - table, - tables, - row, - finalRows, - relationships, - opts - ) + if (!opts?.sqs) { + finalRows = await updateRelationshipColumns( + table, + tables, + row, + finalRows, + relationships, + opts + ) + } } // make sure all related rows are correct diff --git a/packages/server/src/sdk/app/rows/search/internal/sqs.ts b/packages/server/src/sdk/app/rows/search/internal/sqs.ts index 6736ff6abf..fb140e3c14 100644 --- a/packages/server/src/sdk/app/rows/search/internal/sqs.ts +++ b/packages/server/src/sdk/app/rows/search/internal/sqs.ts @@ -37,9 +37,9 @@ import { outputProcessing } from "../../../../../utilities/rowProcessor" import pick from "lodash/pick" import { processRowCountResponse } from "../../utils" import { - updateFilterKeys, getRelationshipColumns, getTableIDList, + updateFilterKeys, } from "../filters" import { dataFilters, @@ -368,6 +368,17 @@ export async function search( }) ) + // make sure relationships have columns reversed correctly + for (let columnName of Object.keys(table.schema)) { + if (table.schema[columnName].type !== FieldType.LINK) { + continue + } + // process the relationships (JSON generated by SQS) + for (let row of processed) { + row[columnName] = reverseUserColumnMapping(row[columnName]) + } + } + // check for pagination final row let nextRow: boolean = false if (paginate && params.limit && rows.length > params.limit) { From 5d53e64360f9832cfee92521d6305b94aff42299 Mon Sep 17 00:00:00 2001 From: mike12345567 Date: Fri, 23 Aug 2024 18:45:13 +0100 Subject: [PATCH 03/85] Getting fields from all relationships loading correctly. --- packages/backend-core/src/sql/sql.ts | 13 +++++++++++-- 1 file changed, 11 insertions(+), 2 deletions(-) diff --git a/packages/backend-core/src/sql/sql.ts b/packages/backend-core/src/sql/sql.ts index 1c87abf5ae..74678a79b5 100644 --- a/packages/backend-core/src/sql/sql.ts +++ b/packages/backend-core/src/sql/sql.ts @@ -761,8 +761,17 @@ class InternalBuilder { const { resource, tableAliases: aliases, endpoint } = this.query const fields = resource?.fields || [] const jsonField = (field: string) => { - const unAliased = field.split(".").slice(1).join(".") - return `'${unAliased}',${field}` + const parts = field.split(".") + let tableField: string, unaliased: string + if (parts.length > 1) { + const alias = parts.shift()! + unaliased = parts.join(".") + tableField = `${this.quote(alias)}.${this.quote(unaliased)}` + } else { + unaliased = parts.join(".") + tableField = this.quote(unaliased) + } + return `'${unaliased}',${tableField}` } for (let relationship of relationships) { const { From b11ee56a38ca8b056f74794c97b5a3366409d3c4 Mon Sep 17 00:00:00 2001 From: mike12345567 Date: Fri, 23 Aug 2024 18:54:46 +0100 Subject: [PATCH 04/85] Adding limit in for wide tables to be related correctly. --- packages/backend-core/src/sql/sql.ts | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/packages/backend-core/src/sql/sql.ts b/packages/backend-core/src/sql/sql.ts index 74678a79b5..5f8a8e1a9b 100644 --- a/packages/backend-core/src/sql/sql.ts +++ b/packages/backend-core/src/sql/sql.ts @@ -39,6 +39,7 @@ import { dataFilters, helpers } from "@budibase/shared-core" import { cloneDeep } from "lodash" type QueryFunction = (query: SqlQuery | SqlQuery[], operation: Operation) => any +const MAX_SQS_RELATIONSHIP_FIELDS = 63 function getBaseLimit() { const envLimit = environment.SQL_MAX_ROWS @@ -800,9 +801,15 @@ class InternalBuilder { alias: throughAlias, schema: endpoint.schema, }) - const relationshipFields = fields.filter( + let relationshipFields = fields.filter( field => field.split(".")[0] === toAlias ) + if (this.client === SqlClient.SQL_LITE) { + relationshipFields = relationshipFields.slice( + 0, + MAX_SQS_RELATIONSHIP_FIELDS + ) + } const fieldList: string = relationshipFields .map(field => jsonField(field)) .join(",") From 0c604b7310a96aba482dab0b25296b5f428fabe3 Mon Sep 17 00:00:00 2001 From: mike12345567 Date: Tue, 27 Aug 2024 18:34:05 +0100 Subject: [PATCH 05/85] Moving things around, making join logic more accessible. --- packages/backend-core/src/sql/sql.ts | 169 ++++++++++++++++++--------- 1 file changed, 112 insertions(+), 57 deletions(-) diff --git a/packages/backend-core/src/sql/sql.ts b/packages/backend-core/src/sql/sql.ts index 5f8a8e1a9b..9fbb02a69e 100644 --- a/packages/backend-core/src/sql/sql.ts +++ b/packages/backend-core/src/sql/sql.ts @@ -337,6 +337,40 @@ class InternalBuilder { return filters } + addRelationshipForFilter( + query: Knex.QueryBuilder, + filterKey: string, + whereCb: (query: Knex.QueryBuilder) => Knex.QueryBuilder + ): Knex.QueryBuilder { + const mainKnex = this.knex + const { relationships, endpoint } = this.query + const tableName = endpoint.entityId + if (!relationships) { + return query + } + for (const relationship of relationships) { + // this is the relationship which is being filtered + if (filterKey.startsWith(relationship.column) && relationship.to) { + const subQuery = query.whereExists(function () { + this.select(mainKnex.raw(1)).from(relationship.tableName!) + }) + query = whereCb( + this.addJoin( + subQuery, + { + from: relationship.tableName, + to: tableName, + through: relationship.through, + }, + [relationship] + ) + ) + break + } + } + return query + } + // right now we only do filters on the specific table being queried addFilters( query: Knex.QueryBuilder, @@ -387,6 +421,7 @@ class InternalBuilder { fn(alias ? `${alias}.${updatedKey}` : updatedKey, value) } if (opts?.relationship && isRelationshipField) { + // TODO: need to update fn to take the query const [filterTableName, property] = updatedKey.split(".") const alias = getTableAlias(filterTableName) fn(alias ? `${alias}.${property}` : property, value) @@ -839,12 +874,75 @@ class InternalBuilder { return query } + addJoin( + query: Knex.QueryBuilder, + tables: { from: string; to: string; through?: string }, + columns: { + from?: string + to?: string + fromPrimary?: string + toPrimary?: string + }[] + ): Knex.QueryBuilder { + const { tableAliases: aliases, endpoint } = this.query + const schema = endpoint.schema + const toTable = tables.to, + fromTable = tables.from, + throughTable = tables.through + const toAlias = aliases?.[toTable] || toTable, + throughAlias = (throughTable && aliases?.[throughTable]) || throughTable, + fromAlias = aliases?.[fromTable] || fromTable + let toTableWithSchema = this.tableNameWithSchema(toTable, { + alias: toAlias, + schema, + }) + let throughTableWithSchema = throughTable + ? this.tableNameWithSchema(throughTable, { + alias: throughAlias, + schema, + }) + : undefined + if (!throughTable) { + // @ts-ignore + query = query.leftJoin(toTableWithSchema, function () { + for (let relationship of columns) { + const from = relationship.from, + to = relationship.to + // @ts-ignore + this.orOn(`${fromAlias}.${from}`, "=", `${toAlias}.${to}`) + } + }) + } else { + query = query + // @ts-ignore + .leftJoin(throughTableWithSchema, function () { + for (let relationship of columns) { + const fromPrimary = relationship.fromPrimary + const from = relationship.from + // @ts-ignore + this.orOn( + `${fromAlias}.${fromPrimary}`, + "=", + `${throughAlias}.${from}` + ) + } + }) + .leftJoin(toTableWithSchema, function () { + for (let relationship of columns) { + const toPrimary = relationship.toPrimary + const to = relationship.to + // @ts-ignore + this.orOn(`${toAlias}.${toPrimary}`, `${throughAlias}.${to}`) + } + }) + } + return query + } + addRelationships( query: Knex.QueryBuilder, fromTable: string, - relationships: RelationshipsJson[], - schema?: string, - aliases?: Record + relationships: RelationshipsJson[] ): Knex.QueryBuilder { const tableSets: Record = {} // aggregate into table sets (all the same to tables) @@ -865,51 +963,15 @@ class InternalBuilder { } for (let [key, relationships] of Object.entries(tableSets)) { const { toTable, throughTable } = JSON.parse(key) - const toAlias = aliases?.[toTable] || toTable, - throughAlias = aliases?.[throughTable] || throughTable, - fromAlias = aliases?.[fromTable] || fromTable - let toTableWithSchema = this.tableNameWithSchema(toTable, { - alias: toAlias, - schema, - }) - let throughTableWithSchema = this.tableNameWithSchema(throughTable, { - alias: throughAlias, - schema, - }) - if (!throughTable) { - // @ts-ignore - query = query.leftJoin(toTableWithSchema, function () { - for (let relationship of relationships) { - const from = relationship.from, - to = relationship.to - // @ts-ignore - this.orOn(`${fromAlias}.${from}`, "=", `${toAlias}.${to}`) - } - }) - } else { - query = query - // @ts-ignore - .leftJoin(throughTableWithSchema, function () { - for (let relationship of relationships) { - const fromPrimary = relationship.fromPrimary - const from = relationship.from - // @ts-ignore - this.orOn( - `${fromAlias}.${fromPrimary}`, - "=", - `${throughAlias}.${from}` - ) - } - }) - .leftJoin(toTableWithSchema, function () { - for (let relationship of relationships) { - const toPrimary = relationship.toPrimary - const to = relationship.to - // @ts-ignore - this.orOn(`${toAlias}.${toPrimary}`, `${throughAlias}.${to}`) - } - }) - } + query = this.addJoin( + query, + { + from: fromTable, + to: toTable, + through: throughTable, + }, + relationships + ) } return query } @@ -1015,8 +1077,7 @@ class InternalBuilder { limits?: { base: number; query: number } } = {} ): Knex.QueryBuilder { - let { endpoint, filters, paginate, relationships, tableAliases } = - this.query + let { endpoint, filters, paginate, relationships } = this.query const { limits } = opts const counting = endpoint.operation === Operation.COUNT @@ -1062,13 +1123,7 @@ class InternalBuilder { if (relationships && this.client === SqlClient.SQL_LITE) { query = this.addJsonRelationships(query, tableName, relationships) } else if (relationships) { - query = this.addRelationships( - query, - tableName, - relationships, - endpoint.schema, - tableAliases - ) + query = this.addRelationships(query, tableName, relationships) } return this.addFilters(query, filters, { relationship: true }) From 49c1f34b5d5fe78fbfd21c5bb80a934149c8e81a Mon Sep 17 00:00:00 2001 From: mike12345567 Date: Wed, 28 Aug 2024 18:41:02 +0100 Subject: [PATCH 06/85] Saving at this point - got exists working. --- packages/backend-core/src/sql/sql.ts | 173 ++++++++++++++++----------- 1 file changed, 101 insertions(+), 72 deletions(-) diff --git a/packages/backend-core/src/sql/sql.ts b/packages/backend-core/src/sql/sql.ts index 9fbb02a69e..1243eaea8a 100644 --- a/packages/backend-core/src/sql/sql.ts +++ b/packages/backend-core/src/sql/sql.ts @@ -343,18 +343,24 @@ class InternalBuilder { whereCb: (query: Knex.QueryBuilder) => Knex.QueryBuilder ): Knex.QueryBuilder { const mainKnex = this.knex - const { relationships, endpoint } = this.query + const { relationships, endpoint, tableAliases: aliases } = this.query const tableName = endpoint.entityId if (!relationships) { return query } for (const relationship of relationships) { // this is the relationship which is being filtered - if (filterKey.startsWith(relationship.column) && relationship.to) { - const subQuery = query.whereExists(function () { - this.select(mainKnex.raw(1)).from(relationship.tableName!) - }) - query = whereCb( + if ( + filterKey.startsWith(`${relationship.tableName}.`) && + relationship.to && + relationship.tableName + ) { + const relatedTableName = relationship.tableName + const alias = aliases?.[relatedTableName] || relatedTableName + let subQuery = mainKnex + .select(mainKnex.raw(1)) + .from({ [alias]: relatedTableName }) + subQuery = whereCb( this.addJoin( subQuery, { @@ -365,6 +371,7 @@ class InternalBuilder { [relationship] ) ) + query = query.whereExists(subQuery) break } } @@ -382,12 +389,13 @@ class InternalBuilder { if (!filters) { return query } + const builder = this filters = this.parseFilters({ ...filters }) const aliases = this.query.tableAliases // if all or specified in filters, then everything is an or const allOr = filters.allOr - const tableName = - this.client === SqlClient.SQL_LITE ? this.table._id! : this.table.name + const isSqlite = this.client === SqlClient.SQL_LITE + const tableName = isSqlite ? this.table._id! : this.table.name function getTableAlias(name: string) { const alias = aliases?.[name] @@ -395,13 +403,33 @@ class InternalBuilder { } function iterate( structure: AnySearchFilter, - fn: (key: string, value: any) => void, - complexKeyFn?: (key: string[], value: any) => void + fn: ( + query: Knex.QueryBuilder, + key: string, + value: any + ) => Knex.QueryBuilder, + complexKeyFn?: ( + query: Knex.QueryBuilder, + key: string[], + value: any + ) => Knex.QueryBuilder ) { + const handleRelationship = ( + q: Knex.QueryBuilder, + key: string, + value: any + ) => { + const [filterTableName, ...otherProperties] = key.split(".") + const property = otherProperties.join(".") + const alias = getTableAlias(filterTableName) + return fn(q, alias ? `${alias}.${property}` : property, value) + } for (const key in structure) { const value = structure[key] const updatedKey = dbCore.removeKeyNumbering(key) const isRelationshipField = updatedKey.includes(".") + const shouldProcessRelationship = + opts?.relationship && isRelationshipField let castedTypeValue if ( @@ -410,7 +438,8 @@ class InternalBuilder { complexKeyFn ) { const alias = getTableAlias(tableName) - complexKeyFn( + query = complexKeyFn( + query, castedTypeValue.id.map((x: string) => alias ? `${alias}.${x}` : x ), @@ -418,27 +447,31 @@ class InternalBuilder { ) } else if (!isRelationshipField) { const alias = getTableAlias(tableName) - fn(alias ? `${alias}.${updatedKey}` : updatedKey, value) - } - if (opts?.relationship && isRelationshipField) { - // TODO: need to update fn to take the query - const [filterTableName, property] = updatedKey.split(".") - const alias = getTableAlias(filterTableName) - fn(alias ? `${alias}.${property}` : property, value) + query = fn( + query, + alias ? `${alias}.${updatedKey}` : updatedKey, + value + ) + } else if (isSqlite && shouldProcessRelationship) { + query = builder.addRelationshipForFilter(query, updatedKey, q => { + return handleRelationship(q, updatedKey, value) + }) + } else if (shouldProcessRelationship) { + query = handleRelationship(query, updatedKey, value) } } } - const like = (key: string, value: any) => { + const like = (q: Knex.QueryBuilder, key: string, value: any) => { const fuzzyOr = filters?.fuzzyOr const fnc = fuzzyOr || allOr ? "orWhere" : "where" // postgres supports ilike, nothing else does if (this.client === SqlClient.POSTGRES) { - query = query[fnc](key, "ilike", `%${value}%`) + return q[fnc](key, "ilike", `%${value}%`) } else { const rawFnc = `${fnc}Raw` // @ts-ignore - query = query[rawFnc](`LOWER(${this.quotedIdentifier(key)}) LIKE ?`, [ + return q[rawFnc](`LOWER(${this.quotedIdentifier(key)}) LIKE ?`, [ `%${value.toLowerCase()}%`, ]) } @@ -456,13 +489,13 @@ class InternalBuilder { return `[${value.join(",")}]` } if (this.client === SqlClient.POSTGRES) { - iterate(mode, (key, value) => { + iterate(mode, (q, key, value) => { const wrap = any ? "" : "'" const op = any ? "\\?| array" : "@>" const fieldNames = key.split(/\./g) const table = fieldNames[0] const col = fieldNames[1] - query = query[rawFnc]( + return q[rawFnc]( `${not}COALESCE("${table}"."${col}"::jsonb ${op} ${wrap}${stringifyArray( value, any ? "'" : '"' @@ -471,8 +504,8 @@ class InternalBuilder { }) } else if (this.client === SqlClient.MY_SQL) { const jsonFnc = any ? "JSON_OVERLAPS" : "JSON_CONTAINS" - iterate(mode, (key, value) => { - query = query[rawFnc]( + iterate(mode, (q, key, value) => { + return q[rawFnc]( `${not}COALESCE(${jsonFnc}(${key}, '${stringifyArray( value )}'), FALSE)` @@ -480,7 +513,7 @@ class InternalBuilder { }) } else { const andOr = mode === filters?.containsAny ? " OR " : " AND " - iterate(mode, (key, value) => { + iterate(mode, (q, key, value) => { let statement = "" const identifier = this.quotedIdentifier(key) for (let i in value) { @@ -495,16 +528,16 @@ class InternalBuilder { } if (statement === "") { - return + return q } if (not) { - query = query[rawFnc]( + return q[rawFnc]( `(NOT (${statement}) OR ${identifier} IS NULL)`, value ) } else { - query = query[rawFnc](statement, value) + return q[rawFnc](statement, value) } }) } @@ -534,39 +567,39 @@ class InternalBuilder { const fnc = allOr ? "orWhereIn" : "whereIn" iterate( filters.oneOf, - (key: string, array) => { + (q, key: string, array) => { if (this.client === SqlClient.ORACLE) { key = this.convertClobs(key) array = Array.isArray(array) ? array : [array] const binding = new Array(array.length).fill("?").join(",") - query = query.whereRaw(`${key} IN (${binding})`, array) + return q.whereRaw(`${key} IN (${binding})`, array) } else { - query = query[fnc](key, Array.isArray(array) ? array : [array]) + return q[fnc](key, Array.isArray(array) ? array : [array]) } }, - (key: string[], array) => { + (q, key: string[], array) => { if (this.client === SqlClient.ORACLE) { const keyStr = `(${key.map(k => this.convertClobs(k)).join(",")})` const binding = `(${array .map((a: any) => `(${new Array(a.length).fill("?").join(",")})`) .join(",")})` - query = query.whereRaw(`${keyStr} IN ${binding}`, array.flat()) + return q.whereRaw(`${keyStr} IN ${binding}`, array.flat()) } else { - query = query[fnc](key, Array.isArray(array) ? array : [array]) + return q[fnc](key, Array.isArray(array) ? array : [array]) } } ) } if (filters.string) { - iterate(filters.string, (key, value) => { + iterate(filters.string, (q, key, value) => { const fnc = allOr ? "orWhere" : "where" // postgres supports ilike, nothing else does if (this.client === SqlClient.POSTGRES) { - query = query[fnc](key, "ilike", `${value}%`) + return q[fnc](key, "ilike", `${value}%`) } else { const rawFnc = `${fnc}Raw` // @ts-ignore - query = query[rawFnc](`LOWER(${this.quotedIdentifier(key)}) LIKE ?`, [ + return q[rawFnc](`LOWER(${this.quotedIdentifier(key)}) LIKE ?`, [ `${value.toLowerCase()}%`, ]) } @@ -576,7 +609,7 @@ class InternalBuilder { iterate(filters.fuzzy, like) } if (filters.range) { - iterate(filters.range, (key, value) => { + iterate(filters.range, (q, key, value) => { const isEmptyObject = (val: any) => { return ( val && @@ -605,97 +638,93 @@ class InternalBuilder { schema?.type === FieldType.BIGINT && this.client === SqlClient.SQL_LITE ) { - query = query.whereRaw( + return q.whereRaw( `CAST(${key} AS INTEGER) BETWEEN CAST(? AS INTEGER) AND CAST(? AS INTEGER)`, [value.low, value.high] ) } else { const fnc = allOr ? "orWhereBetween" : "whereBetween" - query = query[fnc](key, [value.low, value.high]) + return q[fnc](key, [value.low, value.high]) } } else if (lowValid) { if ( schema?.type === FieldType.BIGINT && this.client === SqlClient.SQL_LITE ) { - query = query.whereRaw( - `CAST(${key} AS INTEGER) >= CAST(? AS INTEGER)`, - [value.low] - ) + return q.whereRaw(`CAST(${key} AS INTEGER) >= CAST(? AS INTEGER)`, [ + value.low, + ]) } else { const fnc = allOr ? "orWhere" : "where" - query = query[fnc](key, ">=", value.low) + return q[fnc](key, ">=", value.low) } } else if (highValid) { if ( schema?.type === FieldType.BIGINT && this.client === SqlClient.SQL_LITE ) { - query = query.whereRaw( - `CAST(${key} AS INTEGER) <= CAST(? AS INTEGER)`, - [value.high] - ) + return q.whereRaw(`CAST(${key} AS INTEGER) <= CAST(? AS INTEGER)`, [ + value.high, + ]) } else { const fnc = allOr ? "orWhere" : "where" - query = query[fnc](key, "<=", value.high) + return q[fnc](key, "<=", value.high) } } + return q }) } if (filters.equal) { - iterate(filters.equal, (key, value) => { + iterate(filters.equal, (q, key, value) => { const fnc = allOr ? "orWhereRaw" : "whereRaw" if (this.client === SqlClient.MS_SQL) { - query = query[fnc]( + return q[fnc]( `CASE WHEN ${this.quotedIdentifier(key)} = ? THEN 1 ELSE 0 END = 1`, [value] ) } else if (this.client === SqlClient.ORACLE) { const identifier = this.convertClobs(key) - query = query[fnc]( - `(${identifier} IS NOT NULL AND ${identifier} = ?)`, - [value] - ) + return q[fnc](`(${identifier} IS NOT NULL AND ${identifier} = ?)`, [ + value, + ]) } else { - query = query[fnc]( - `COALESCE(${this.quotedIdentifier(key)} = ?, FALSE)`, - [value] - ) + return q[fnc](`COALESCE(${this.quotedIdentifier(key)} = ?, FALSE)`, [ + value, + ]) } }) } if (filters.notEqual) { - iterate(filters.notEqual, (key, value) => { + iterate(filters.notEqual, (q, key, value) => { const fnc = allOr ? "orWhereRaw" : "whereRaw" if (this.client === SqlClient.MS_SQL) { - query = query[fnc]( + return q[fnc]( `CASE WHEN ${this.quotedIdentifier(key)} = ? THEN 1 ELSE 0 END = 0`, [value] ) } else if (this.client === SqlClient.ORACLE) { const identifier = this.convertClobs(key) - query = query[fnc]( + return q[fnc]( `(${identifier} IS NOT NULL AND ${identifier} != ?) OR ${identifier} IS NULL`, [value] ) } else { - query = query[fnc]( - `COALESCE(${this.quotedIdentifier(key)} != ?, TRUE)`, - [value] - ) + return q[fnc](`COALESCE(${this.quotedIdentifier(key)} != ?, TRUE)`, [ + value, + ]) } }) } if (filters.empty) { - iterate(filters.empty, key => { + iterate(filters.empty, (q, key) => { const fnc = allOr ? "orWhereNull" : "whereNull" - query = query[fnc](key) + return q[fnc](key) }) } if (filters.notEmpty) { - iterate(filters.notEmpty, key => { + iterate(filters.notEmpty, (q, key) => { const fnc = allOr ? "orWhereNotNull" : "whereNotNull" - query = query[fnc](key) + return q[fnc](key) }) } if (filters.contains) { From 628964364a3ce5e7c6a988656b8f0fa535d15c07 Mon Sep 17 00:00:00 2001 From: mike12345567 Date: Wed, 28 Aug 2024 18:55:15 +0100 Subject: [PATCH 07/85] Getting through join working as expected. --- packages/backend-core/src/sql/sql.ts | 40 +++++++++++++++++++--------- 1 file changed, 28 insertions(+), 12 deletions(-) diff --git a/packages/backend-core/src/sql/sql.ts b/packages/backend-core/src/sql/sql.ts index 1243eaea8a..84231288a2 100644 --- a/packages/backend-core/src/sql/sql.ts +++ b/packages/backend-core/src/sql/sql.ts @@ -345,6 +345,7 @@ class InternalBuilder { const mainKnex = this.knex const { relationships, endpoint, tableAliases: aliases } = this.query const tableName = endpoint.entityId + const fromAlias = aliases?.[tableName] || tableName if (!relationships) { return query } @@ -356,22 +357,37 @@ class InternalBuilder { relationship.tableName ) { const relatedTableName = relationship.tableName - const alias = aliases?.[relatedTableName] || relatedTableName + const toAlias = aliases?.[relatedTableName] || relatedTableName let subQuery = mainKnex .select(mainKnex.raw(1)) - .from({ [alias]: relatedTableName }) - subQuery = whereCb( - this.addJoin( - subQuery, - { - from: relationship.tableName, - to: tableName, - through: relationship.through, - }, - [relationship] + .from({ [toAlias]: relatedTableName }) + let mainTableRelatesTo = toAlias + if (relationship.through) { + const throughAlias = + aliases?.[relationship.through] || relationship.through + let throughTable = this.tableNameWithSchema(relationship.through, { + alias: throughAlias, + schema: endpoint.schema, + }) + subQuery = subQuery.innerJoin(throughTable, function () { + // @ts-ignore + this.orOn( + `${toAlias}.${relationship.toPrimary}`, + "=", + `${throughAlias}.${relationship.to}` + ) + }) + mainTableRelatesTo = throughAlias + } + // "join" to the main table, making sure the ID matches that of the main + subQuery = subQuery.where( + `${mainTableRelatesTo}.${relationship.from}`, + "=", + mainKnex.raw( + this.quotedIdentifier(`${fromAlias}.${relationship.fromPrimary}`) ) ) - query = query.whereExists(subQuery) + query = query.whereExists(whereCb(subQuery)) break } } From 3e51dde6d2be09351faaed3e6c79f214977d65ff Mon Sep 17 00:00:00 2001 From: mike12345567 Date: Thu, 29 Aug 2024 17:58:11 +0100 Subject: [PATCH 08/85] Check for alias as well when deciding whether filter requires relationship addition. --- packages/backend-core/src/sql/sql.ts | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/packages/backend-core/src/sql/sql.ts b/packages/backend-core/src/sql/sql.ts index 84231288a2..9b6819d433 100644 --- a/packages/backend-core/src/sql/sql.ts +++ b/packages/backend-core/src/sql/sql.ts @@ -346,21 +346,25 @@ class InternalBuilder { const { relationships, endpoint, tableAliases: aliases } = this.query const tableName = endpoint.entityId const fromAlias = aliases?.[tableName] || tableName + const matches = (possibleTable: string) => + filterKey.startsWith(`${possibleTable}`) if (!relationships) { return query } for (const relationship of relationships) { + const relatedTableName = relationship.tableName + const toAlias = aliases?.[relatedTableName] || relatedTableName // this is the relationship which is being filtered if ( - filterKey.startsWith(`${relationship.tableName}.`) && + (matches(relatedTableName) || matches(toAlias)) && relationship.to && relationship.tableName ) { - const relatedTableName = relationship.tableName - const toAlias = aliases?.[relatedTableName] || relatedTableName let subQuery = mainKnex .select(mainKnex.raw(1)) .from({ [toAlias]: relatedTableName }) + // relationships should never have more than the base limit + .limit(getBaseLimit()) let mainTableRelatesTo = toAlias if (relationship.through) { const throughAlias = From a9b1a2240332e7ea337ed0e2c6625b97228ab2ad Mon Sep 17 00:00:00 2001 From: mike12345567 Date: Thu, 29 Aug 2024 18:56:14 +0100 Subject: [PATCH 09/85] Some improvements to get SQS tests passing. --- packages/backend-core/src/sql/sql.ts | 20 +++++++++++++++---- .../src/api/routes/tests/search.spec.ts | 3 ++- .../src/utilities/rowProcessor/index.ts | 7 +++++++ 3 files changed, 25 insertions(+), 5 deletions(-) diff --git a/packages/backend-core/src/sql/sql.ts b/packages/backend-core/src/sql/sql.ts index 9b6819d433..e6738d4b36 100644 --- a/packages/backend-core/src/sql/sql.ts +++ b/packages/backend-core/src/sql/sql.ts @@ -337,6 +337,11 @@ class InternalBuilder { return filters } + addJoinFieldCheck(query: Knex.QueryBuilder, relationship: RelationshipsJson) { + const document = relationship.from?.split(".")[0] || "" + return query.andWhere(`${document}.fieldName`, "=", relationship.column) + } + addRelationshipForFilter( query: Knex.QueryBuilder, filterKey: string, @@ -363,8 +368,6 @@ class InternalBuilder { let subQuery = mainKnex .select(mainKnex.raw(1)) .from({ [toAlias]: relatedTableName }) - // relationships should never have more than the base limit - .limit(getBaseLimit()) let mainTableRelatesTo = toAlias if (relationship.through) { const throughAlias = @@ -375,12 +378,15 @@ class InternalBuilder { }) subQuery = subQuery.innerJoin(throughTable, function () { // @ts-ignore - this.orOn( + this.on( `${toAlias}.${relationship.toPrimary}`, "=", `${throughAlias}.${relationship.to}` ) }) + if (this.client === SqlClient.SQL_LITE) { + subQuery = this.addJoinFieldCheck(subQuery, relationship) + } mainTableRelatesTo = throughAlias } // "join" to the main table, making sure the ID matches that of the main @@ -907,7 +913,7 @@ class InternalBuilder { default: throw new Error(`JSON relationships not implement for ${this.client}`) } - const subQuery = this.knex + let subQuery = this.knex .select(rawJsonArray) .from(toTableWithSchema) .join(throughTableWithSchema, function () { @@ -918,6 +924,12 @@ class InternalBuilder { "=", this.knex.raw(this.quotedIdentifier(`${fromAlias}.${fromPrimary}`)) ) + // relationships should never have more than the base limit + .limit(getBaseLimit()) + // need to check the junction table document is to the right column + if (this.client === SqlClient.SQL_LITE) { + subQuery = this.addJoinFieldCheck(subQuery, relationship) + } query = query.select({ [relationship.column]: subQuery }) } return query diff --git a/packages/server/src/api/routes/tests/search.spec.ts b/packages/server/src/api/routes/tests/search.spec.ts index bac9b6f774..44fe18d0ef 100644 --- a/packages/server/src/api/routes/tests/search.spec.ts +++ b/packages/server/src/api/routes/tests/search.spec.ts @@ -2687,7 +2687,8 @@ describe.each([ }) }) - isSql && + // TODO: when all SQL databases use the same mechanism - remove this test, new relationship system doesn't have this problem + !isInternal && describe("pagination edge case with relationships", () => { let mainRows: Row[] = [] diff --git a/packages/server/src/utilities/rowProcessor/index.ts b/packages/server/src/utilities/rowProcessor/index.ts index 795f6970ab..6028b2d248 100644 --- a/packages/server/src/utilities/rowProcessor/index.ts +++ b/packages/server/src/utilities/rowProcessor/index.ts @@ -336,6 +336,13 @@ export async function outputProcessing( row[property] = `${hours}:${minutes}:${seconds}` } } + } else if (column.type === FieldType.LINK) { + for (let row of enriched) { + // if relationship is empty - remove the array, this has been part of the API for some time + if (Array.isArray(row[property]) && row[property].length === 0) { + delete row[property] + } + } } } From 67301054913024ecab6e0bc5650ee464c73cd34d Mon Sep 17 00:00:00 2001 From: mike12345567 Date: Fri, 30 Aug 2024 13:44:23 +0100 Subject: [PATCH 10/85] Adding the option to disable user sync, always importing large apps which are problematic. --- packages/server/src/environment.ts | 1 + packages/server/src/sdk/app/applications/sync.ts | 10 ++++++++-- 2 files changed, 9 insertions(+), 2 deletions(-) diff --git a/packages/server/src/environment.ts b/packages/server/src/environment.ts index 05ebbe3151..585eb6a7f2 100644 --- a/packages/server/src/environment.ts +++ b/packages/server/src/environment.ts @@ -112,6 +112,7 @@ const environment = { parseIntSafe(process.env.JS_RUNNER_MEMORY_LIMIT) || DEFAULTS.JS_RUNNER_MEMORY_LIMIT, LOG_JS_ERRORS: process.env.LOG_JS_ERRORS, + DISABLE_USER_SYNC: process.env.DISABLE_USER_SYNC, // old CLIENT_ID: process.env.CLIENT_ID, _set(key: string, value: any) { diff --git a/packages/server/src/sdk/app/applications/sync.ts b/packages/server/src/sdk/app/applications/sync.ts index 44e44a5aaa..37450acf1d 100644 --- a/packages/server/src/sdk/app/applications/sync.ts +++ b/packages/server/src/sdk/app/applications/sync.ts @@ -8,6 +8,10 @@ import { generateUserMetadataID, InternalTables } from "../../../db/utils" type DeletedUser = { _id: string; deleted: boolean } +function userSyncEnabled() { + return !env.DISABLE_USER_SYNC +} + async function syncUsersToApp( appId: string, users: (User | DeletedUser)[], @@ -56,7 +60,7 @@ async function syncUsersToApp( // the user doesn't exist, or doesn't have a role anymore // get rid of their metadata - if (deletedUser || !roleId) { + if (userSyncEnabled() && (deletedUser || !roleId)) { await db.remove(metadata) continue } @@ -149,7 +153,9 @@ export async function syncApp( } // sync the users - kept for safe keeping - await sdk.users.syncGlobalUsers() + if (userSyncEnabled()) { + await sdk.users.syncGlobalUsers() + } if (error) { throw error From 1a57e37d38502e12b88ac625d475e656388f74de Mon Sep 17 00:00:00 2001 From: Martin McKeaveney Date: Mon, 2 Sep 2024 15:22:33 +0100 Subject: [PATCH 11/85] initial framework for platform AI --- .../portal/settings/ai/ConfigModal.svelte | 81 +++++++++++++++++++ .../builder/portal/settings/ai/index.svelte | 74 +++++++++++++++++ packages/builder/src/stores/portal/menu.js | 4 + 3 files changed, 159 insertions(+) create mode 100644 packages/builder/src/pages/builder/portal/settings/ai/ConfigModal.svelte create mode 100644 packages/builder/src/pages/builder/portal/settings/ai/index.svelte diff --git a/packages/builder/src/pages/builder/portal/settings/ai/ConfigModal.svelte b/packages/builder/src/pages/builder/portal/settings/ai/ConfigModal.svelte new file mode 100644 index 0000000000..408e71bd1f --- /dev/null +++ b/packages/builder/src/pages/builder/portal/settings/ai/ConfigModal.svelte @@ -0,0 +1,81 @@ + + + {}} + disabled={!validation} + size="M" + title="Custom AI Configuration" +> +
+ + +
+
+ + +
+
+ + +
+
+ + +
+ + + + + diff --git a/packages/builder/src/pages/builder/portal/settings/ai/index.svelte b/packages/builder/src/pages/builder/portal/settings/ai/index.svelte new file mode 100644 index 0000000000..e40a6cf4ff --- /dev/null +++ b/packages/builder/src/pages/builder/portal/settings/ai/index.svelte @@ -0,0 +1,74 @@ + + + + + + + + + + + AI + Configure your AI settings within this section: + + + +
+ AI Configurations + +
+ Use the following interface to select your preferred AI configuration. + Select your AI Model: + + + + diff --git a/packages/builder/src/stores/portal/menu.js b/packages/builder/src/stores/portal/menu.js index c7c71d89bc..ec40f6d4ee 100644 --- a/packages/builder/src/stores/portal/menu.js +++ b/packages/builder/src/stores/portal/menu.js @@ -45,6 +45,10 @@ export const menu = derived([admin, auth], ([$admin, $auth]) => { title: "Auth", href: "/builder/portal/settings/auth", }, + { + title: "AI", + href: "/builder/portal/settings/ai", + }, { title: "Email", href: "/builder/portal/settings/email", From 99035ad8aea269ef93c6e26cd892c167503797a9 Mon Sep 17 00:00:00 2001 From: Martin McKeaveney Date: Mon, 2 Sep 2024 20:16:37 +0100 Subject: [PATCH 12/85] rough pass on AI platform CRUD --- packages/backend-core/src/configs/configs.ts | 41 +++++++ .../portal/settings/ai/ConfigModal.svelte | 106 +++++++++++++----- .../builder/portal/settings/ai/ai.spec.ts | 0 .../builder/portal/settings/ai/constants.ts | 8 ++ .../builder/portal/settings/ai/index.svelte | 69 +++++++----- .../builder/src/stores/portal/licensing.js | 12 ++ .../functions/backfill/global/configs.ts | 7 +- packages/types/src/documents/global/config.ts | 18 +++ packages/types/src/sdk/licensing/feature.ts | 2 + .../src/api/controllers/global/configs.ts | 14 +++ 10 files changed, 220 insertions(+), 57 deletions(-) create mode 100644 packages/builder/src/pages/builder/portal/settings/ai/ai.spec.ts create mode 100644 packages/builder/src/pages/builder/portal/settings/ai/constants.ts diff --git a/packages/backend-core/src/configs/configs.ts b/packages/backend-core/src/configs/configs.ts index 0d189e3f7d..6062d3624f 100644 --- a/packages/backend-core/src/configs/configs.ts +++ b/packages/backend-core/src/configs/configs.ts @@ -1,4 +1,5 @@ import { + AIConfig, AIInnerConfig, Config, ConfigType, GoogleConfig, @@ -17,6 +18,7 @@ import { DocumentType, SEPARATOR } from "../constants" import { CacheKey, TTL, withCache } from "../cache" import * as context from "../context" import env from "../environment" +import { getConfigParams } from "@budibase/server/src/migrations/functions/backfill/global/configs" // UTILS @@ -254,3 +256,42 @@ export async function getSCIMConfig(): Promise { const config = await getConfig(ConfigType.SCIM) return config?.config } + + +// AI + +// TODO: Can we assume that you are licensed when you hit this endpoint? +export async function getAIConfig(): Promise { + if (!env.SELF_HOSTED) { + // always use the env vars in cloud + // TODO: Licensing stuff - make this right + if (env.OPENAI_API_KEY) { + return getDefaultBudibaseAIConfig() + } + } + + // prefer the config in self-host + let config = await getConfig(ConfigType.AI) + + // fallback to env vars + if (!config || !config.activated) { + config = getDefaultBudibaseAIConfig() + } + + return config +} + +export function getDefaultBudibaseAIConfig(): AIInnerConfig | undefined { + if (env.OPENAI_API_KEY) { + return { + provider: "", + isDefault: true, + name: "Budibase AI", + active: true, + baseUrl: "", + apiKey: env.OPENAI_API_KEY, + // TODO: should be enum + defaultModel: "" + } + } +} diff --git a/packages/builder/src/pages/builder/portal/settings/ai/ConfigModal.svelte b/packages/builder/src/pages/builder/portal/settings/ai/ConfigModal.svelte index 408e71bd1f..30830cff1a 100644 --- a/packages/builder/src/pages/builder/portal/settings/ai/ConfigModal.svelte +++ b/packages/builder/src/pages/builder/portal/settings/ai/ConfigModal.svelte @@ -5,15 +5,14 @@ Input, Select, Toggle, - Dropzone, Body, notifications, } from "@budibase/bbui" - import KeyValueBuilder from "components/integration/KeyValueBuilder.svelte" - import { plugins } from "stores/portal" - import { PluginSource } from "constants" + import { ConfigMap } from "./constants" + import { API } from "api" - const options = [ + // TODO: Update these + const providers = [ "OpenAI", "Anthropic", "Together AI", @@ -21,21 +20,69 @@ "Custom" ] - let source = PluginSource.GITHUB - let dynamicValues = {} + const models = [ + "gpt4o-mini", + ] - let name - let baseUrl - let apiKey - let active - let isDefault + let formValues = {} let validation + + $: { + const { provider, model, name, apiKey } = formValues + validation = provider && model && name && apiKey + } + + function prefillConfig(evt) { + const provider = evt.detail + // grab the preset config from the constants for that provider and fill it in + if (ConfigMap[provider]) { + formValues = { + ...formValues, + ...ConfigMap[provider] + } + } else { + formValues = { + provider + } + } + } + + async function saveConfig() { + try { + const savedConfig = await API.saveConfig(formValues) + formValues._rev = savedConfig._rev + formValues._id = savedConfig._id + notifications.success(`Configuration saved`) + } catch (error) { + notifications.error( + `Failed to save AI Configuration, reason: ${error?.message || "Unknown"}` + ) + } + } + + async function deleteConfig() { + // Delete a configuration + try { + // await API.deleteConfig({ + // id: smtpConfig._id, + // rev: smtpConfig._rev, + // }) + notifications.success(`Deleted config`) + } catch (error) { + notifications.error( + `Failed to clear email settings, reason: ${error?.message || "Unknown"}` + ) + } + } + {}} + cancelText={"Delete"} + onConfirm={saveConfig} + onCancel={deleteConfig} disabled={!validation} size="M" title="Custom AI Configuration" @@ -43,39 +90,40 @@
- +
- +
- +
- - + + diff --git a/packages/builder/src/pages/builder/portal/settings/ai/ai.spec.ts b/packages/builder/src/pages/builder/portal/settings/ai/ai.spec.ts new file mode 100644 index 0000000000..e69de29bb2 diff --git a/packages/builder/src/pages/builder/portal/settings/ai/constants.ts b/packages/builder/src/pages/builder/portal/settings/ai/constants.ts new file mode 100644 index 0000000000..df5508109d --- /dev/null +++ b/packages/builder/src/pages/builder/portal/settings/ai/constants.ts @@ -0,0 +1,8 @@ +export const ConfigMap = { + OpenAI: { + baseUrl: "https://api.openai.com" + }, + Anthropic: { + baseUrl: "https://api.together.xyz/v1" + }, +} \ No newline at end of file diff --git a/packages/builder/src/pages/builder/portal/settings/ai/index.svelte b/packages/builder/src/pages/builder/portal/settings/ai/index.svelte index e40a6cf4ff..427b09a7f2 100644 --- a/packages/builder/src/pages/builder/portal/settings/ai/index.svelte +++ b/packages/builder/src/pages/builder/portal/settings/ai/index.svelte @@ -8,57 +8,76 @@ Divider, notifications, Modal, - ModalContent, + Tags, + Tag } from "@budibase/bbui" - import { auth, admin } from "stores/portal" - import { redirect } from "@roxi/routify" + import { admin, licensing } from "stores/portal" import { API } from "api" import AIConfigModal from "./ConfigModal.svelte" import { onMount } from "svelte" import { sdk } from "@budibase/shared-core" - let diagnosticInfo = "" - let modal + const ConfigTypes = { + AI: "ai", + } - // async function fetchSystemDebugInfo() { - // const diagnostics = await API.fetchSystemDebugInfo() - // diagnosticInfo = { - // browser: { - // language: navigator.language || navigator.userLanguage, - // userAgent: navigator.userAgent, - // platform: navigator.platform, - // vendor: navigator.vendor, - // }, - // server: diagnostics, - // } - // } - // - // const copyToClipboard = async () => { - // await Helpers.copyToClipboard(JSON.stringify(diagnosticInfo, undefined, 2)) - // notifications.success("Copied") - // } + let modal + let aiConfig + let loading = false + + $: isCloud = $admin.cloud + $: budibaseAIEnabled = $licensing.budibaseAIEnabled + $: customAIConfigsEnabled = $licensing.customAIConfigsEnabled + + async function fetchAIConfig() { + loading = true + try { + // Fetch the configs for smtp + const aiDoc = await API.getConfig(ConfigTypes.AI) + if (aiDoc._id) { + aiConfig = aiDoc + } + loading = false + } catch (error) { + notifications.error("Error fetching AI config") + } + } onMount(async () => { - // await fetchSystemDebugInfo() + await fetchAIConfig() }) - AI + {#if isCloud && !budibaseAIEnabled} + + Premium + + {/if} Configure your AI settings within this section:
AI Configurations - + {#if !isCloud && !customAIConfigsEnabled} + + Premium + + {:else if isCloud && !customAIConfigsEnabled} + + Enterprise + + {:else} + + {/if}
Use the following interface to select your preferred AI configuration. Select your AI Model: diff --git a/packages/builder/src/stores/portal/licensing.js b/packages/builder/src/stores/portal/licensing.js index 0e44650479..d48377207c 100644 --- a/packages/builder/src/stores/portal/licensing.js +++ b/packages/builder/src/stores/portal/licensing.js @@ -22,6 +22,8 @@ export const createLicensingStore = () => { backupsEnabled: false, brandingEnabled: false, scimEnabled: false, + budibaseAIEnabled: false, + customAIConfigsEnabled: false, // the currently used quotas from the db quotaUsage: undefined, // derived quota metrics for percentages used @@ -142,6 +144,14 @@ export const createLicensingStore = () => { Constants.Features.VIEW_READONLY_COLUMNS ) + const budibaseAIEnabled = license.features.includes( + Constants.Features.BUDIBASE_AI + ) + + const customAIConfigsEnabled = license.features.includes( + Constants.Features.AI_CUSTOM_CONFIGS + ) + store.update(state => { return { ...state, @@ -153,6 +163,8 @@ export const createLicensingStore = () => { groupsEnabled, backupsEnabled, brandingEnabled, + budibaseAIEnabled, + customAIConfigsEnabled, scimEnabled, environmentVariablesEnabled, auditLogsEnabled, diff --git a/packages/server/src/migrations/functions/backfill/global/configs.ts b/packages/server/src/migrations/functions/backfill/global/configs.ts index 04eb9caff2..bb0781eba3 100644 --- a/packages/server/src/migrations/functions/backfill/global/configs.ts +++ b/packages/server/src/migrations/functions/backfill/global/configs.ts @@ -15,11 +15,12 @@ import { } from "@budibase/types" import env from "./../../../../environment" -export function getConfigParams(): DatabaseQueryOpts { +export function getConfigParams(type?: ConfigType): DatabaseQueryOpts { + const configType = type || "" return { include_docs: true, - startkey: `${DocumentType.CONFIG}${SEPARATOR}`, - endkey: `${DocumentType.CONFIG}${SEPARATOR}${UNICODE_MAX}`, + startkey: `${DocumentType.CONFIG}${SEPARATOR}${configType}`, + endkey: `${DocumentType.CONFIG}${SEPARATOR}${configType}${UNICODE_MAX}`, } } diff --git a/packages/types/src/documents/global/config.ts b/packages/types/src/documents/global/config.ts index 3fd352aada..246874d0f1 100644 --- a/packages/types/src/documents/global/config.ts +++ b/packages/types/src/documents/global/config.ts @@ -111,6 +111,20 @@ export interface SCIMInnerConfig { export interface SCIMConfig extends Config {} +export interface AIInnerConfig { + // TODO: should be enum + provider: string + isDefault: boolean + name: string + active: boolean + baseUrl: string + apiKey: string + // TODO: should be enum + defaultModel: string +} + +export interface AIConfig extends Config {} + export const isSettingsConfig = (config: Config): config is SettingsConfig => config.type === ConfigType.SETTINGS @@ -126,6 +140,9 @@ export const isOIDCConfig = (config: Config): config is OIDCConfig => export const isSCIMConfig = (config: Config): config is SCIMConfig => config.type === ConfigType.SCIM +export const isAIConfig = (config: Config): config is AIConfig => + config.type === ConfigType.AI + export enum ConfigType { SETTINGS = "settings", ACCOUNT = "account", @@ -134,4 +151,5 @@ export enum ConfigType { OIDC = "oidc", OIDC_LOGOS = "logos_oidc", SCIM = "scim", + AI = "ai" } diff --git a/packages/types/src/sdk/licensing/feature.ts b/packages/types/src/sdk/licensing/feature.ts index 0d8db8258e..bf2bcb72b6 100644 --- a/packages/types/src/sdk/licensing/feature.ts +++ b/packages/types/src/sdk/licensing/feature.ts @@ -15,6 +15,8 @@ export enum Feature { EXPANDED_PUBLIC_API = "expandedPublicApi", VIEW_PERMISSIONS = "viewPermissions", VIEW_READONLY_COLUMNS = "viewReadonlyColumns", + BUDIBASE_AI = "budibaseAI", + AI_CUSTOM_CONFIGS = "customAIConfigs" } export type PlanFeatures = { [key in PlanType]: Feature[] | undefined } diff --git a/packages/worker/src/api/controllers/global/configs.ts b/packages/worker/src/api/controllers/global/configs.ts index a44b173869..43aac19b18 100644 --- a/packages/worker/src/api/controllers/global/configs.ts +++ b/packages/worker/src/api/controllers/global/configs.ts @@ -197,6 +197,12 @@ async function verifyOIDCConfig(config: OIDCConfigs) { await verifySSOConfig(ConfigType.OIDC, config.configs[0]) } +async function verifyAIConfig(config: OIDCConfigs) { + // await verifySSOConfig(ConfigType.OIDC, config.configs[0]) + // Shape should be `config_ai` + return true +} + export async function save(ctx: UserCtx) { const body = ctx.request.body const type = body.type @@ -224,6 +230,9 @@ export async function save(ctx: UserCtx) { case ConfigType.OIDC: await verifyOIDCConfig(config) break + case ConfigType.AI: + await verifyAIConfig(config) + break } } catch (err: any) { ctx.throw(400, err) @@ -314,6 +323,11 @@ export async function find(ctx: UserCtx) { if (type === ConfigType.OIDC_LOGOS) { enrichOIDCLogos(scopedConfig) } + + if (type === ConfigType.AI) { + // TODO: strip the keys from the configs here + // TODO: do the licensing checks here and return the right things based on the license + } ctx.body = scopedConfig } else { // don't throw an error, there simply is nothing to return From 67d5219f754ad15ed313a4c88a2ea2621e61b15f Mon Sep 17 00:00:00 2001 From: Martin McKeaveney Date: Mon, 2 Sep 2024 20:20:58 +0100 Subject: [PATCH 13/85] basic crud working --- .../src/pages/builder/portal/settings/ai/ConfigModal.svelte | 1 + 1 file changed, 1 insertion(+) diff --git a/packages/builder/src/pages/builder/portal/settings/ai/ConfigModal.svelte b/packages/builder/src/pages/builder/portal/settings/ai/ConfigModal.svelte index 30830cff1a..e9ec5159f9 100644 --- a/packages/builder/src/pages/builder/portal/settings/ai/ConfigModal.svelte +++ b/packages/builder/src/pages/builder/portal/settings/ai/ConfigModal.svelte @@ -49,6 +49,7 @@ } async function saveConfig() { + formValues.type = "ai" try { const savedConfig = await API.saveConfig(formValues) formValues._rev = savedConfig._rev From f328ae4bf9dff216e8a23471cc60b074f0e41e0d Mon Sep 17 00:00:00 2001 From: Martin McKeaveney Date: Mon, 2 Sep 2024 20:45:32 +0100 Subject: [PATCH 14/85] account for arrays --- packages/backend-core/src/configs/configs.ts | 3 +- .../portal/settings/ai/ConfigModal.svelte | 49 ++++++++++++------- .../worker/src/api/routes/global/configs.ts | 20 +++++++- 3 files changed, 51 insertions(+), 21 deletions(-) diff --git a/packages/backend-core/src/configs/configs.ts b/packages/backend-core/src/configs/configs.ts index 6062d3624f..1771716f57 100644 --- a/packages/backend-core/src/configs/configs.ts +++ b/packages/backend-core/src/configs/configs.ts @@ -1,5 +1,6 @@ import { - AIConfig, AIInnerConfig, + AIConfig, + AIInnerConfig, Config, ConfigType, GoogleConfig, diff --git a/packages/builder/src/pages/builder/portal/settings/ai/ConfigModal.svelte b/packages/builder/src/pages/builder/portal/settings/ai/ConfigModal.svelte index e9ec5159f9..749cebeb2c 100644 --- a/packages/builder/src/pages/builder/portal/settings/ai/ConfigModal.svelte +++ b/packages/builder/src/pages/builder/portal/settings/ai/ConfigModal.svelte @@ -23,37 +23,48 @@ const models = [ "gpt4o-mini", ] + + const defaultConfig = { + active: false, + isDefault: false, + } - let formValues = {} - + let aiConfig = defaultConfig let validation $: { - const { provider, model, name, apiKey } = formValues - validation = provider && model && name && apiKey + const { provider, defaultModel, name, apiKey } = aiConfig + validation = provider && defaultModel && name && apiKey } function prefillConfig(evt) { const provider = evt.detail // grab the preset config from the constants for that provider and fill it in if (ConfigMap[provider]) { - formValues = { - ...formValues, + aiConfig = { + ...aiConfig, ...ConfigMap[provider] } } else { - formValues = { + aiConfig = { + ...aiConfig, provider } } } async function saveConfig() { - formValues.type = "ai" + const config = { + type: "ai", + config: [ + // TODO: include the ones that are already there, or just handle this in the backend + aiConfig, + ] + } try { - const savedConfig = await API.saveConfig(formValues) - formValues._rev = savedConfig._rev - formValues._id = savedConfig._id + const savedConfig = await API.saveConfig(config) + aiConfig._rev = savedConfig._rev + aiConfig._id = savedConfig._id notifications.success(`Configuration saved`) } catch (error) { notifications.error( @@ -92,33 +103,33 @@
- +
- +
- +
- - + + diff --git a/packages/builder/src/pages/builder/portal/settings/ai/ConfigModal.svelte b/packages/builder/src/pages/builder/portal/settings/ai/ConfigModal.svelte index 749cebeb2c..70ddba5407 100644 --- a/packages/builder/src/pages/builder/portal/settings/ai/ConfigModal.svelte +++ b/packages/builder/src/pages/builder/portal/settings/ai/ConfigModal.svelte @@ -8,27 +8,17 @@ Body, notifications, } from "@budibase/bbui" - import { ConfigMap } from "./constants" + import { ConfigMap, Providers } from "./constants" import { API } from "api" - // TODO: Update these - const providers = [ - "OpenAI", - "Anthropic", - "Together AI", - "Azure Open AI", - "Custom" - ] - - const models = [ - "gpt4o-mini", - ] - - const defaultConfig = { + export let defaultConfig = { active: false, isDefault: false, } + export let saveHandler + export let deleteHandler + let aiConfig = defaultConfig let validation @@ -43,48 +33,15 @@ if (ConfigMap[provider]) { aiConfig = { ...aiConfig, - ...ConfigMap[provider] - } - } else { - aiConfig = { - ...aiConfig, + ...ConfigMap[provider], provider } - } - } - - async function saveConfig() { - const config = { - type: "ai", - config: [ - // TODO: include the ones that are already there, or just handle this in the backend - aiConfig, - ] - } - try { - const savedConfig = await API.saveConfig(config) - aiConfig._rev = savedConfig._rev - aiConfig._id = savedConfig._id - notifications.success(`Configuration saved`) - } catch (error) { - notifications.error( - `Failed to save AI Configuration, reason: ${error?.message || "Unknown"}` - ) - } - } - - async function deleteConfig() { - // Delete a configuration - try { - // await API.deleteConfig({ - // id: smtpConfig._id, - // rev: smtpConfig._rev, - // }) - notifications.success(`Deleted config`) - } catch (error) { - notifications.error( - `Failed to clear email settings, reason: ${error?.message || "Unknown"}` - ) + } else { + aiConfig.provider = provider + // aiConfig = { + // ...aiConfig, + // provider + // } } } @@ -93,8 +50,8 @@
+
diff --git a/packages/builder/src/pages/builder/portal/settings/ai/constants.ts b/packages/builder/src/pages/builder/portal/settings/ai/constants.ts index df5508109d..07e0ea1cb7 100644 --- a/packages/builder/src/pages/builder/portal/settings/ai/constants.ts +++ b/packages/builder/src/pages/builder/portal/settings/ai/constants.ts @@ -1,8 +1,47 @@ +export const Providers = { + OpenAI: { + name: "OpenAI", + models: [ + "gpt-4o-mini", + "gpt-4o", + "gpt-3.5-turbo", + "chatgpt-4o-latest", + "gpt-4-turbo", + "gpt-4", + ] + }, + Anthropic: { + name: "Anthropic", + models: [ + "claude-3-5-sonnet-20240620", + "claude-3-sonnet-20240229", + "claude-3-opus-20240229", + "claude-3-haiku-20240307" + ] + }, + TogetherAI: { + name: "Together AI", + // TODO: too many - probably need to use an autocomplete for this + models: [""] + }, + Custom: { + name: "Custom", + // TODO: too many - probably need to use an autocomplete for this + models: [""] + }, +} + export const ConfigMap = { OpenAI: { baseUrl: "https://api.openai.com" }, Anthropic: { + baseUrl: "" + }, + TogetherAI: { baseUrl: "https://api.together.xyz/v1" }, + Custom: { + baseUrl: "" + } } \ No newline at end of file diff --git a/packages/builder/src/pages/builder/portal/settings/ai/index.svelte b/packages/builder/src/pages/builder/portal/settings/ai/index.svelte index 427b09a7f2..65cd137833 100644 --- a/packages/builder/src/pages/builder/portal/settings/ai/index.svelte +++ b/packages/builder/src/pages/builder/portal/settings/ai/index.svelte @@ -1,4 +1,5 @@ + + + + + + \ No newline at end of file diff --git a/packages/builder/src/pages/builder/portal/settings/ai/logos/OpenAI.svelte b/packages/builder/src/pages/builder/portal/settings/ai/logos/OpenAI.svelte new file mode 100644 index 0000000000..a7a2927dbc --- /dev/null +++ b/packages/builder/src/pages/builder/portal/settings/ai/logos/OpenAI.svelte @@ -0,0 +1,5 @@ + +OpenAI icon \ No newline at end of file diff --git a/packages/types/src/documents/global/config.ts b/packages/types/src/documents/global/config.ts index 246874d0f1..249893520c 100644 --- a/packages/types/src/documents/global/config.ts +++ b/packages/types/src/documents/global/config.ts @@ -123,7 +123,7 @@ export interface AIInnerConfig { defaultModel: string } -export interface AIConfig extends Config {} +export interface AIConfig extends Config {} export const isSettingsConfig = (config: Config): config is SettingsConfig => config.type === ConfigType.SETTINGS diff --git a/packages/worker/src/api/controllers/global/configs.ts b/packages/worker/src/api/controllers/global/configs.ts index 43aac19b18..e5c142f562 100644 --- a/packages/worker/src/api/controllers/global/configs.ts +++ b/packages/worker/src/api/controllers/global/configs.ts @@ -28,7 +28,7 @@ import { SSOConfig, SSOConfigType, UserCtx, - OIDCLogosConfig, + OIDCLogosConfig, AIConfig, } from "@budibase/types" import * as pro from "@budibase/pro" @@ -313,6 +313,13 @@ function enrichOIDCLogos(oidcLogos: OIDCLogosConfig) { ) } +function sanitizeAIConfig(aiConfig: AIConfig) { + for (let providerConfig of aiConfig.config) { + delete providerConfig.apiKey + } + return aiConfig +} + export async function find(ctx: UserCtx) { try { // Find the config with the most granular scope based on context @@ -325,8 +332,8 @@ export async function find(ctx: UserCtx) { } if (type === ConfigType.AI) { - // TODO: strip the keys from the configs here // TODO: do the licensing checks here and return the right things based on the license + sanitizeAIConfig(scopedConfig) } ctx.body = scopedConfig } else { diff --git a/packages/worker/src/api/routes/global/configs.ts b/packages/worker/src/api/routes/global/configs.ts index b857647652..417498d829 100644 --- a/packages/worker/src/api/routes/global/configs.ts +++ b/packages/worker/src/api/routes/global/configs.ts @@ -73,7 +73,7 @@ function aiValidation() { isDefault: Joi.boolean().required(), name: Joi.string().required(), active: Joi.boolean().required(), - baseUrl: Joi.string().optional(), + baseUrl: Joi.string().optional().allow("", null), apiKey: Joi.string().required(), // TODO: should be enum defaultModel: Joi.string().optional(), From 43135e42746032672cd59299e86ec9e80f1cf3ac Mon Sep 17 00:00:00 2001 From: Martin McKeaveney Date: Tue, 3 Sep 2024 17:18:46 +0100 Subject: [PATCH 17/85] AI Config CRUD complete --- .../portal/settings/ai/AIConfigTile.svelte | 2 +- .../portal/settings/ai/ConfigModal.svelte | 61 +++++++--------- .../builder/portal/settings/ai/index.svelte | 69 ++++++++++--------- .../src/api/controllers/global/configs.ts | 20 ++++-- .../worker/src/api/routes/global/configs.ts | 26 +++---- 5 files changed, 91 insertions(+), 87 deletions(-) diff --git a/packages/builder/src/pages/builder/portal/settings/ai/AIConfigTile.svelte b/packages/builder/src/pages/builder/portal/settings/ai/AIConfigTile.svelte index 12cfa1d174..4b9b4cc506 100644 --- a/packages/builder/src/pages/builder/portal/settings/ai/AIConfigTile.svelte +++ b/packages/builder/src/pages/builder/portal/settings/ai/AIConfigTile.svelte @@ -1,5 +1,5 @@
+
+ + +
-
- - -
- +
- +
- - + + diff --git a/packages/builder/src/pages/builder/portal/settings/ai/constants.ts b/packages/builder/src/pages/builder/portal/settings/ai/constants.ts index 07e0ea1cb7..9c82f714a4 100644 --- a/packages/builder/src/pages/builder/portal/settings/ai/constants.ts +++ b/packages/builder/src/pages/builder/portal/settings/ai/constants.ts @@ -22,7 +22,7 @@ export const Providers = { TogetherAI: { name: "Together AI", // TODO: too many - probably need to use an autocomplete for this - models: [""] + models: ["whatever"] }, Custom: { name: "Custom", diff --git a/packages/builder/src/pages/builder/portal/settings/ai/logos/Budibase.svelte b/packages/builder/src/pages/builder/portal/settings/ai/logos/Budibase.svelte new file mode 100644 index 0000000000..f69c7d8281 --- /dev/null +++ b/packages/builder/src/pages/builder/portal/settings/ai/logos/Budibase.svelte @@ -0,0 +1,17 @@ + + + + + + + + + + + + + + diff --git a/packages/builder/src/pages/builder/portal/settings/ai/logos/TogetherAI.svelte b/packages/builder/src/pages/builder/portal/settings/ai/logos/TogetherAI.svelte new file mode 100644 index 0000000000..1afd5626b1 --- /dev/null +++ b/packages/builder/src/pages/builder/portal/settings/ai/logos/TogetherAI.svelte @@ -0,0 +1,17 @@ + + + + + + + + + + + + + + \ No newline at end of file From 378ccb6e70b48695e05490ec7d29fc0c8603253a Mon Sep 17 00:00:00 2001 From: Martin McKeaveney Date: Tue, 3 Sep 2024 18:52:33 +0100 Subject: [PATCH 22/85] switch name and description --- .../src/pages/builder/portal/settings/ai/AIConfigTile.svelte | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/builder/src/pages/builder/portal/settings/ai/AIConfigTile.svelte b/packages/builder/src/pages/builder/portal/settings/ai/AIConfigTile.svelte index d02c2c62c3..1b2b4b99a7 100644 --- a/packages/builder/src/pages/builder/portal/settings/ai/AIConfigTile.svelte +++ b/packages/builder/src/pages/builder/portal/settings/ai/AIConfigTile.svelte @@ -25,7 +25,7 @@ {/if}
- {config.name} + {config.provider}
From 0d5e776424b5c12a9588ffbc2f921b33d39e4e8b Mon Sep 17 00:00:00 2001 From: Martin McKeaveney Date: Tue, 3 Sep 2024 19:12:13 +0100 Subject: [PATCH 23/85] preserve apikeys on save --- .../portal/settings/ai/ConfigModal.svelte | 3 ++- .../builder/portal/settings/ai/index.svelte | 10 ++++----- packages/types/src/documents/global/config.ts | 22 ++++++++++--------- .../src/api/controllers/global/configs.ts | 11 ++++++---- 4 files changed, 25 insertions(+), 21 deletions(-) diff --git a/packages/builder/src/pages/builder/portal/settings/ai/ConfigModal.svelte b/packages/builder/src/pages/builder/portal/settings/ai/ConfigModal.svelte index 703f1e8129..747a070c6b 100644 --- a/packages/builder/src/pages/builder/portal/settings/ai/ConfigModal.svelte +++ b/packages/builder/src/pages/builder/portal/settings/ai/ConfigModal.svelte @@ -22,6 +22,7 @@ const {provider, defaultModel, name, apiKey} = config validation = provider && defaultModel && name && apiKey } + $: canEditBaseUrl = config.provider && ConfigMap[config.provider].baseUrl === "" function prefillConfig(evt) { const provider = evt.detail @@ -70,7 +71,7 @@
- +
diff --git a/packages/builder/src/pages/builder/portal/settings/ai/index.svelte b/packages/builder/src/pages/builder/portal/settings/ai/index.svelte index d0ced7041a..57435168ac 100644 --- a/packages/builder/src/pages/builder/portal/settings/ai/index.svelte +++ b/packages/builder/src/pages/builder/portal/settings/ai/index.svelte @@ -60,10 +60,9 @@ } try { - const savedConfig = await API.saveConfig(fullAIConfig) - fullAIConfig._rev = savedConfig._rev - fullAIConfig._id = savedConfig._id + await API.saveConfig(fullAIConfig) notifications.success(`Successfully saved and activated AI Configuration`) + await fetchAIConfig() } catch (error) { notifications.error( `Failed to save AI Configuration, reason: ${error?.message || "Unknown"}` @@ -76,10 +75,9 @@ delete fullAIConfig.config[key] try { - const savedConfig = await API.saveConfig(fullAIConfig) - fullAIConfig._rev = savedConfig._rev - fullAIConfig._id = savedConfig._id + await API.saveConfig(fullAIConfig) notifications.success(`Deleted config`) + await fetchAIConfig() } catch (error) { notifications.error( `Failed to delete config, reason: ${error?.message || "Unknown"}` diff --git a/packages/types/src/documents/global/config.ts b/packages/types/src/documents/global/config.ts index 249893520c..35caedef63 100644 --- a/packages/types/src/documents/global/config.ts +++ b/packages/types/src/documents/global/config.ts @@ -112,18 +112,20 @@ export interface SCIMInnerConfig { export interface SCIMConfig extends Config {} export interface AIInnerConfig { - // TODO: should be enum - provider: string - isDefault: boolean - name: string - active: boolean - baseUrl: string - apiKey: string - // TODO: should be enum - defaultModel: string + [key: string]: { + // TODO: should be enum + provider: string + isDefault: boolean + name: string + active: boolean + baseUrl: string + apiKey: string + // TODO: should be enum + defaultModel: string + } } -export interface AIConfig extends Config {} +export interface AIConfig extends Config {} export const isSettingsConfig = (config: Config): config is SettingsConfig => config.type === ConfigType.SETTINGS diff --git a/packages/worker/src/api/controllers/global/configs.ts b/packages/worker/src/api/controllers/global/configs.ts index 0b8e525eb8..1b66289f47 100644 --- a/packages/worker/src/api/controllers/global/configs.ts +++ b/packages/worker/src/api/controllers/global/configs.ts @@ -200,11 +200,14 @@ async function verifyOIDCConfig(config: OIDCConfigs) { } async function verifyAIConfig(config: AIConfig, existingConfig?: AIConfig) { + if (!existingConfig) return + // ensure that the redacted API keys are not overwritten in the DB - // for (let aiConfig of existingConfig) { - // - // } - return true + for (let uuid in existingConfig.config) { + if (config[uuid]?.apiKey === PASSWORD_REPLACEMENT) { + config[uuid].apiKey = existingConfig.config[uuid].apiKey + } + } } export async function save(ctx: UserCtx) { From be4b6aaceb47a1b6048e9c6b4cd9edf007b090d6 Mon Sep 17 00:00:00 2001 From: Martin McKeaveney Date: Tue, 3 Sep 2024 19:19:33 +0100 Subject: [PATCH 24/85] adding config for azure openai and more customisation for custom ai models --- .../builder/portal/settings/ai/ConfigModal.svelte | 14 +++++++++----- .../pages/builder/portal/settings/ai/constants.ts | 11 +++++++++++ 2 files changed, 20 insertions(+), 5 deletions(-) diff --git a/packages/builder/src/pages/builder/portal/settings/ai/ConfigModal.svelte b/packages/builder/src/pages/builder/portal/settings/ai/ConfigModal.svelte index 747a070c6b..cac8f74786 100644 --- a/packages/builder/src/pages/builder/portal/settings/ai/ConfigModal.svelte +++ b/packages/builder/src/pages/builder/portal/settings/ai/ConfigModal.svelte @@ -63,11 +63,15 @@
- + {:else} + + {/if}
diff --git a/packages/builder/src/pages/builder/portal/settings/ai/constants.ts b/packages/builder/src/pages/builder/portal/settings/ai/constants.ts index 9c82f714a4..d0db0d0d9b 100644 --- a/packages/builder/src/pages/builder/portal/settings/ai/constants.ts +++ b/packages/builder/src/pages/builder/portal/settings/ai/constants.ts @@ -22,6 +22,17 @@ export const Providers = { TogetherAI: { name: "Together AI", // TODO: too many - probably need to use an autocomplete for this + models: [ + "gpt-4o-mini", + "gpt-4o", + "gpt-3.5-turbo", + "chatgpt-4o-latest", + "gpt-4-turbo", + "gpt-4", + ] + }, + AzureOpenAI: { + name: "Azure Open AI", models: ["whatever"] }, Custom: { From d3b90a02539c50a1dfb7a97ca6b52d103407bce3 Mon Sep 17 00:00:00 2001 From: Martin McKeaveney Date: Tue, 3 Sep 2024 19:47:36 +0100 Subject: [PATCH 25/85] initial impl of default Budibase AI config through env vars --- .../portal/settings/ai/AIConfigTile.svelte | 6 +++++- .../portal/settings/ai/ConfigModal.svelte | 10 +++++----- .../builder/portal/settings/ai/constants.ts | 19 +++++++++++++------ packages/types/src/documents/global/config.ts | 10 +++++----- .../src/api/controllers/global/configs.ts | 19 ++++++++++++++++--- packages/worker/src/environment.ts | 3 +++ 6 files changed, 47 insertions(+), 20 deletions(-) diff --git a/packages/builder/src/pages/builder/portal/settings/ai/AIConfigTile.svelte b/packages/builder/src/pages/builder/portal/settings/ai/AIConfigTile.svelte index 1b2b4b99a7..39c0972027 100644 --- a/packages/builder/src/pages/builder/portal/settings/ai/AIConfigTile.svelte +++ b/packages/builder/src/pages/builder/portal/settings/ai/AIConfigTile.svelte @@ -1,5 +1,6 @@ - - - export let height - export let width + export let height + export let width -OpenAI icon \ No newline at end of file + + OpenAI icon + + \ No newline at end of file diff --git a/packages/types/src/sdk/licensing/quota.ts b/packages/types/src/sdk/licensing/quota.ts index 85700f167b..748d90fda7 100644 --- a/packages/types/src/sdk/licensing/quota.ts +++ b/packages/types/src/sdk/licensing/quota.ts @@ -17,12 +17,14 @@ export enum StaticQuotaName { CREATORS = "creators", USER_GROUPS = "userGroups", PLUGINS = "plugins", + AI_CUSTOM_CONFIGS = "aiCustomConfigs", } export enum MonthlyQuotaName { QUERIES = "queries", AUTOMATIONS = "automations", DAY_PASSES = "dayPasses", + BUDIBASE_AI_CREDITS = "budibaseAICredits" } export enum ConstantQuotaName { @@ -62,6 +64,7 @@ export type MonthlyQuotas = { [MonthlyQuotaName.QUERIES]: Quota [MonthlyQuotaName.AUTOMATIONS]: Quota [MonthlyQuotaName.DAY_PASSES]: Quota + [MonthlyQuotaName.BUDIBASE_AI_CREDITS]: Quota } export type StaticQuotas = { @@ -71,6 +74,7 @@ export type StaticQuotas = { [StaticQuotaName.CREATORS]: Quota [StaticQuotaName.USER_GROUPS]: Quota [StaticQuotaName.PLUGINS]: Quota + [StaticQuotaName.AI_CUSTOM_CONFIGS]: Quota } export type ConstantQuotas = { From fed82dffaf0e62997d6d030e0df134b4e6071248 Mon Sep 17 00:00:00 2001 From: mike12345567 Date: Wed, 4 Sep 2024 13:11:03 +0100 Subject: [PATCH 28/85] Linting. --- packages/backend-core/src/sql/sql.ts | 2 +- .../src/api/controllers/row/utils/sqlUtils.ts | 16 ---------------- 2 files changed, 1 insertion(+), 17 deletions(-) diff --git a/packages/backend-core/src/sql/sql.ts b/packages/backend-core/src/sql/sql.ts index 6181d18046..0bc9126924 100644 --- a/packages/backend-core/src/sql/sql.ts +++ b/packages/backend-core/src/sql/sql.ts @@ -128,7 +128,7 @@ class InternalBuilder { } private generateSelectStatement(): (string | Knex.Raw)[] | "*" { - const { endpoint, resource, meta, tableAliases } = this.query + const { endpoint, resource, tableAliases } = this.query if (!resource || !resource.fields || resource.fields.length === 0) { return "*" diff --git a/packages/server/src/api/controllers/row/utils/sqlUtils.ts b/packages/server/src/api/controllers/row/utils/sqlUtils.ts index d6e5c3e8f1..249bb43bbc 100644 --- a/packages/server/src/api/controllers/row/utils/sqlUtils.ts +++ b/packages/server/src/api/controllers/row/utils/sqlUtils.ts @@ -7,11 +7,9 @@ import { ManyToManyRelationshipFieldMetadata, RelationshipFieldMetadata, RelationshipsJson, - Row, Table, } from "@budibase/types" import { breakExternalTableId } from "../../../../integrations/utils" -import { basicProcessing } from "./basic" import { generateJunctionTableID } from "../../../../db/utils" type TableMap = Record @@ -22,20 +20,6 @@ export function isManyToMany( return !!(field as ManyToManyRelationshipFieldMetadata).through } -function isCorrectRelationship( - relationship: RelationshipsJson, - table1: Table, - table2: Table, - row: Row -): boolean { - const junctionTableId = generateJunctionTableID(table1._id!, table2._id!) - const possibleColumns = [ - `${junctionTableId}.doc1.fieldName`, - `${junctionTableId}.doc2.fieldName`, - ] - return !!possibleColumns.find(col => row[col] === relationship.column) -} - /** * Gets the list of relationship JSON structures based on the columns in the table, * this will be used by the underlying library to build whatever relationship mechanism From eefb1f01a3521a052f8b0c16ef6366053cbb6319 Mon Sep 17 00:00:00 2001 From: mike12345567 Date: Wed, 4 Sep 2024 13:18:54 +0100 Subject: [PATCH 29/85] Fix for generic sql test. --- .../src/api/routes/tests/queries/generic-sql.spec.ts | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/packages/server/src/api/routes/tests/queries/generic-sql.spec.ts b/packages/server/src/api/routes/tests/queries/generic-sql.spec.ts index a84b243e2d..4bd1951d67 100644 --- a/packages/server/src/api/routes/tests/queries/generic-sql.spec.ts +++ b/packages/server/src/api/routes/tests/queries/generic-sql.spec.ts @@ -832,10 +832,12 @@ describe.each( }, }) expect(res).toHaveLength(1) - expect(res[0]).toEqual({ - id: 2, - name: "two", - }) + expect(res[0]).toEqual( + expect.objectContaining({ + id: 2, + name: "two", + }) + ) }) // this parameter really only impacts SQL queries From 2553432ec960656212d951a53420b7eee6eec6ed Mon Sep 17 00:00:00 2001 From: Sam Rose Date: Wed, 4 Sep 2024 14:21:25 +0100 Subject: [PATCH 30/85] wip --- .../server/src/integrations/googlesheets.ts | 4 +- .../integrations/tests/googlesheets.spec.ts | 98 +++++++++++-------- 2 files changed, 58 insertions(+), 44 deletions(-) diff --git a/packages/server/src/integrations/googlesheets.ts b/packages/server/src/integrations/googlesheets.ts index f8f5209890..b257a5da9b 100644 --- a/packages/server/src/integrations/googlesheets.ts +++ b/packages/server/src/integrations/googlesheets.ts @@ -31,7 +31,7 @@ import { cache, configs, context, HTTPError } from "@budibase/backend-core" import { dataFilters, utils } from "@budibase/shared-core" import { GOOGLE_SHEETS_PRIMARY_KEY } from "../constants" -interface GoogleSheetsConfig { +export interface GoogleSheetsConfig { spreadsheetId: string auth: OAuthClientConfig continueSetupId?: string @@ -157,7 +157,7 @@ const SCHEMA: Integration = { }, } -class GoogleSheetsIntegration implements DatasourcePlus { +export class GoogleSheetsIntegration implements DatasourcePlus { private readonly config: GoogleSheetsConfig private readonly spreadsheetId: string private client: GoogleSpreadsheet = undefined! diff --git a/packages/server/src/integrations/tests/googlesheets.spec.ts b/packages/server/src/integrations/tests/googlesheets.spec.ts index e9d5290b2c..685345660d 100644 --- a/packages/server/src/integrations/tests/googlesheets.spec.ts +++ b/packages/server/src/integrations/tests/googlesheets.spec.ts @@ -2,49 +2,48 @@ import { setEnv as setCoreEnv } from "@budibase/backend-core" import type { GoogleSpreadsheetWorksheet } from "google-spreadsheet" import nock from "nock" -jest.mock("google-auth-library") -const { OAuth2Client } = require("google-auth-library") - -const setCredentialsMock = jest.fn() -const getAccessTokenMock = jest.fn() - -OAuth2Client.mockImplementation(() => { - return { - setCredentials: setCredentialsMock, - getAccessToken: getAccessTokenMock, - } -}) - -jest.mock("google-spreadsheet") -const { GoogleSpreadsheet } = require("google-spreadsheet") - -const sheetsByTitle: { [title: string]: GoogleSpreadsheetWorksheet } = {} -const sheetsByIndex: GoogleSpreadsheetWorksheet[] = [] -const mockGoogleIntegration = { - useOAuth2Client: jest.fn(), - loadInfo: jest.fn(), - sheetsByTitle, - sheetsByIndex, -} - -GoogleSpreadsheet.mockImplementation(() => mockGoogleIntegration) - import { structures } from "@budibase/backend-core/tests" import TestConfiguration from "../../tests/utilities/TestConfiguration" -import GoogleSheetsIntegration from "../googlesheets" -import { FieldType, Table, TableSchema, TableSourceType } from "@budibase/types" +import { GoogleSheetsConfig, GoogleSheetsIntegration } from "../googlesheets" +import { + Datasource, + FieldType, + SourceName, + Table, + TableSchema, + TableSourceType, +} from "@budibase/types" import { generateDatasourceID } from "../../db/utils" describe("Google Sheets Integration", () => { - let integration: any, - config = new TestConfiguration() - let cleanupEnv: () => void + const config = new TestConfiguration() - beforeAll(() => { + let integration: GoogleSheetsIntegration + let cleanupEnv: () => void + let table: Table + let datasource: Datasource + + const datasourceConfig: GoogleSheetsConfig = { + spreadsheetId: "randomId", + auth: { + appId: "appId", + accessToken: "accessToken", + refreshToken: "refreshToken", + }, + } + + beforeAll(async () => { cleanupEnv = setCoreEnv({ GOOGLE_CLIENT_ID: "test", GOOGLE_CLIENT_SECRET: "test", }) + + datasource = await config.api.datasource.create({ + name: "Test Datasource", + type: "datasource", + source: SourceName.GOOGLE_SHEETS, + config: datasourceConfig, + }) }) afterAll(async () => { @@ -53,17 +52,32 @@ describe("Google Sheets Integration", () => { }) beforeEach(async () => { - integration = new GoogleSheetsIntegration.integration({ - spreadsheetId: "randomId", - auth: { - appId: "appId", - accessToken: "accessToken", - refreshToken: "refreshToken", - }, - }) await config.init() - jest.clearAllMocks() + integration = new GoogleSheetsIntegration(datasourceConfig) + + table = await config.api.table.save({ + name: "Test Table", + type: "table", + sourceId: generateDatasourceID(), + sourceType: TableSourceType.EXTERNAL, + schema: { + name: { + name: "name", + type: FieldType.STRING, + constraints: { + type: "string", + }, + }, + description: { + name: "description", + type: FieldType.STRING, + constraints: { + type: "string", + }, + }, + }, + }) nock.cleanAll() nock("https://www.googleapis.com/").post("/oauth2/v4/token").reply(200, { From 79de7b2c4578d82eba71e3643c48422e849548b0 Mon Sep 17 00:00:00 2001 From: mike12345567 Date: Wed, 4 Sep 2024 16:17:25 +0100 Subject: [PATCH 31/85] Updating to use a sub-query with a wrapper to get the JSON aggregations. --- packages/backend-core/src/sql/sql.ts | 20 ++++++++------------ 1 file changed, 8 insertions(+), 12 deletions(-) diff --git a/packages/backend-core/src/sql/sql.ts b/packages/backend-core/src/sql/sql.ts index 0bc9126924..7b11f4ce37 100644 --- a/packages/backend-core/src/sql/sql.ts +++ b/packages/backend-core/src/sql/sql.ts @@ -913,30 +913,26 @@ class InternalBuilder { const fieldList: string = relationshipFields .map(field => jsonField(field)) .join(",") - let rawJsonArray: Knex.Raw, limit: number + let rawJsonArray: Knex.Raw switch (sqlClient) { case SqlClient.SQL_LITE: rawJsonArray = this.knex.raw( `json_group_array(json_object(${fieldList}))` ) - limit = getBaseLimit() break case SqlClient.POSTGRES: rawJsonArray = this.knex.raw( `json_agg(json_build_object(${fieldList}))` ) - limit = 1 break case SqlClient.MY_SQL: case SqlClient.ORACLE: rawJsonArray = this.knex.raw( `json_arrayagg(json_object(${fieldList}))` ) - limit = getBaseLimit() break case SqlClient.MS_SQL: rawJsonArray = this.knex.raw(`json_array(json_object(${fieldList}))`) - limit = 1 break default: throw new Error(`JSON relationships not implement for ${this.client}`) @@ -945,16 +941,12 @@ class InternalBuilder { // it reduces the result set rather than limiting how much data it filters over const primaryKey = `${toAlias}.${toPrimary || toKey}` let subQuery = this.knex - .select(rawJsonArray) + .select(`${toAlias}.*`) .from(toTableWithSchema) - .limit(limit) + .limit(getBaseLimit()) // add sorting to get consistent order .orderBy(primaryKey) - if (sqlClient === SqlClient.POSTGRES) { - subQuery = subQuery.groupBy(primaryKey) - } - // many-to-many relationship with junction table if (throughTable && toPrimary && fromPrimary) { const throughAlias = aliases?.[throughTable] || throughTable @@ -985,7 +977,11 @@ class InternalBuilder { if (this.client === SqlClient.SQL_LITE) { subQuery = this.addJoinFieldCheck(subQuery, relationship) } - query = query.select({ [relationship.column]: subQuery }) + // @ts-ignore - the from alias syntax isn't in Knex typing + const wrapperQuery = this.knex.select(rawJsonArray).from({ + [toAlias]: subQuery, + }) + query = query.select({ [relationship.column]: wrapperQuery }) } return query } From 12db64513ba2597397daf2db9fedcab71d875c71 Mon Sep 17 00:00:00 2001 From: Sam Rose Date: Wed, 4 Sep 2024 16:21:32 +0100 Subject: [PATCH 32/85] Revert to testing against mssql 2017, attempt to get relationship aggreggation working. --- packages/backend-core/src/sql/sql.ts | 30 ++++++++++--------- packages/server/datasource-sha.env | 2 +- .../src/integrations/microsoftSqlServer.ts | 4 +-- 3 files changed, 19 insertions(+), 17 deletions(-) diff --git a/packages/backend-core/src/sql/sql.ts b/packages/backend-core/src/sql/sql.ts index 0bc9126924..b51079ac2d 100644 --- a/packages/backend-core/src/sql/sql.ts +++ b/packages/backend-core/src/sql/sql.ts @@ -913,30 +913,25 @@ class InternalBuilder { const fieldList: string = relationshipFields .map(field => jsonField(field)) .join(",") - let rawJsonArray: Knex.Raw, limit: number + let select: Knex.Raw, limit: number switch (sqlClient) { case SqlClient.SQL_LITE: - rawJsonArray = this.knex.raw( - `json_group_array(json_object(${fieldList}))` - ) + select = this.knex.raw(`json_group_array(json_object(${fieldList}))`) limit = getBaseLimit() break case SqlClient.POSTGRES: - rawJsonArray = this.knex.raw( - `json_agg(json_build_object(${fieldList}))` - ) + select = this.knex.raw(`json_agg(json_build_object(${fieldList}))`) limit = 1 break case SqlClient.MY_SQL: case SqlClient.ORACLE: - rawJsonArray = this.knex.raw( - `json_arrayagg(json_object(${fieldList}))` - ) + select = this.knex.raw(`json_arrayagg(json_object(${fieldList}))`) limit = getBaseLimit() break case SqlClient.MS_SQL: - rawJsonArray = this.knex.raw(`json_array(json_object(${fieldList}))`) - limit = 1 + // Cursed, needs some code later instead + select = this.knex.raw(`*`) + limit = getBaseLimit() break default: throw new Error(`JSON relationships not implement for ${this.client}`) @@ -944,8 +939,8 @@ class InternalBuilder { // SQL Server uses TOP - which performs a little differently to the normal LIMIT syntax // it reduces the result set rather than limiting how much data it filters over const primaryKey = `${toAlias}.${toPrimary || toKey}` - let subQuery = this.knex - .select(rawJsonArray) + let subQuery: Knex.QueryBuilder | Knex.Raw = this.knex + .select(select) .from(toTableWithSchema) .limit(limit) // add sorting to get consistent order @@ -985,6 +980,13 @@ class InternalBuilder { if (this.client === SqlClient.SQL_LITE) { subQuery = this.addJoinFieldCheck(subQuery, relationship) } + + if (this.client === SqlClient.MS_SQL) { + subQuery = this.knex.raw( + `(SELECT a.* FROM (${subQuery}) AS a FOR JSON PATH)` + ) + } + query = query.select({ [relationship.column]: subQuery }) } return query diff --git a/packages/server/datasource-sha.env b/packages/server/datasource-sha.env index 9b935ed8eb..61249d530c 100644 --- a/packages/server/datasource-sha.env +++ b/packages/server/datasource-sha.env @@ -1,4 +1,4 @@ -MSSQL_SHA=sha256:c4369c38385eba011c10906dc8892425831275bb035d5ce69656da8e29de50d8 +MSSQL_SHA=sha256:3b913841850a4d57fcfcb798be06acc88ea0f2acc5418bc0c140a43e91c4a545 MYSQL_SHA=sha256:9de9d54fecee6253130e65154b930978b1fcc336bcc86dfd06e89b72a2588ebe POSTGRES_SHA=sha256:bd0d8e485d1aca439d39e5ea99b931160bd28d862e74c786f7508e9d0053090e MONGODB_SHA=sha256:afa36bca12295b5f9dae68a493c706113922bdab520e901bd5d6c9d7247a1d8d diff --git a/packages/server/src/integrations/microsoftSqlServer.ts b/packages/server/src/integrations/microsoftSqlServer.ts index 88c75891e6..0a07371cd3 100644 --- a/packages/server/src/integrations/microsoftSqlServer.ts +++ b/packages/server/src/integrations/microsoftSqlServer.ts @@ -343,9 +343,9 @@ class SqlServerIntegration extends Sql implements DatasourcePlus { err.number ) if (readableMessage) { - throw new Error(readableMessage) + throw new Error(readableMessage, { cause: err }) } else { - throw new Error(err.message as string) + throw new Error(err.message as string, { cause: err }) } } } From cda778598d8e4e9cb77f7a34bfc932b0ee4939ca Mon Sep 17 00:00:00 2001 From: Sam Rose Date: Wed, 4 Sep 2024 16:41:36 +0100 Subject: [PATCH 33/85] Fix some MSSQL test cases. --- packages/backend-core/src/sql/sql.ts | 14 ++++++++------ 1 file changed, 8 insertions(+), 6 deletions(-) diff --git a/packages/backend-core/src/sql/sql.ts b/packages/backend-core/src/sql/sql.ts index 60cfdea9d9..b283ed2106 100644 --- a/packages/backend-core/src/sql/sql.ts +++ b/packages/backend-core/src/sql/sql.ts @@ -973,16 +973,18 @@ class InternalBuilder { subQuery = this.addJoinFieldCheck(subQuery, relationship) } + let wrapperQuery: Knex.QueryBuilder | Knex.Raw if (this.client === SqlClient.MS_SQL) { - subQuery = this.knex.raw( - `(SELECT a.* FROM (${subQuery}) AS a FOR JSON PATH)` + wrapperQuery = this.knex.raw( + `(SELECT [${toAlias}] = (SELECT a.* FROM (${subQuery}) AS a FOR JSON PATH))` ) + } else { + // @ts-ignore - the from alias syntax isn't in Knex typing + wrapperQuery = this.knex.select(select).from({ + [toAlias]: subQuery, + }) } - // @ts-ignore - the from alias syntax isn't in Knex typing - const wrapperQuery = this.knex.select(select).from({ - [toAlias]: subQuery, - }) query = query.select({ [relationship.column]: wrapperQuery }) } return query From 637ac55a9f87f10bf2f9e954667888cd68f81383 Mon Sep 17 00:00:00 2001 From: mike12345567 Date: Wed, 4 Sep 2024 17:42:30 +0100 Subject: [PATCH 34/85] Slight refactor. --- packages/backend-core/src/sql/sql.ts | 73 +++++++++++++++------------- 1 file changed, 39 insertions(+), 34 deletions(-) diff --git a/packages/backend-core/src/sql/sql.ts b/packages/backend-core/src/sql/sql.ts index b283ed2106..5dd94456f4 100644 --- a/packages/backend-core/src/sql/sql.ts +++ b/packages/backend-core/src/sql/sql.ts @@ -859,6 +859,7 @@ class InternalBuilder { relationships: RelationshipsJson[] ): Knex.QueryBuilder { const sqlClient = this.client + const knex = this.knex const { resource, tableAliases: aliases, endpoint } = this.query const fields = resource?.fields || [] const jsonField = (field: string) => { @@ -913,29 +914,10 @@ class InternalBuilder { const fieldList: string = relationshipFields .map(field => jsonField(field)) .join(",") - let select: Knex.Raw - switch (sqlClient) { - case SqlClient.SQL_LITE: - select = this.knex.raw(`json_group_array(json_object(${fieldList}))`) - break - case SqlClient.POSTGRES: - select = this.knex.raw(`json_agg(json_build_object(${fieldList}))`) - break - case SqlClient.MY_SQL: - case SqlClient.ORACLE: - select = this.knex.raw(`json_arrayagg(json_object(${fieldList}))`) - break - case SqlClient.MS_SQL: - // Cursed, needs some code later instead - select = this.knex.raw(`*`) - break - default: - throw new Error(`JSON relationships not implement for ${this.client}`) - } // SQL Server uses TOP - which performs a little differently to the normal LIMIT syntax // it reduces the result set rather than limiting how much data it filters over const primaryKey = `${toAlias}.${toPrimary || toKey}` - let subQuery: Knex.QueryBuilder | Knex.Raw = this.knex + let subQuery: Knex.QueryBuilder | Knex.Raw = knex .select(`${toAlias}.*`) .from(toTableWithSchema) .limit(getBaseLimit()) @@ -956,7 +938,7 @@ class InternalBuilder { .where( `${throughAlias}.${fromKey}`, "=", - this.knex.raw(this.quotedIdentifier(`${fromAlias}.${fromPrimary}`)) + knex.raw(this.quotedIdentifier(`${fromAlias}.${fromPrimary}`)) ) } // one-to-many relationship with foreign key @@ -964,26 +946,49 @@ class InternalBuilder { subQuery = subQuery.where( `${toAlias}.${toKey}`, "=", - this.knex.raw(this.quotedIdentifier(`${fromAlias}.${fromKey}`)) + knex.raw(this.quotedIdentifier(`${fromAlias}.${fromKey}`)) ) } - // need to check the junction table document is to the right column, this is just for SQS - if (this.client === SqlClient.SQL_LITE) { - subQuery = this.addJoinFieldCheck(subQuery, relationship) - } - - let wrapperQuery: Knex.QueryBuilder | Knex.Raw - if (this.client === SqlClient.MS_SQL) { - wrapperQuery = this.knex.raw( - `(SELECT [${toAlias}] = (SELECT a.* FROM (${subQuery}) AS a FOR JSON PATH))` - ) - } else { + const standardWrap = (select: string): Knex.QueryBuilder => { // @ts-ignore - the from alias syntax isn't in Knex typing - wrapperQuery = this.knex.select(select).from({ + return knex.select(knex.raw(select)).from({ [toAlias]: subQuery, }) } + let wrapperQuery: Knex.QueryBuilder | Knex.Raw + switch (sqlClient) { + case SqlClient.SQL_LITE: + // need to check the junction table document is to the right column, this is just for SQS + subQuery = this.addJoinFieldCheck(subQuery, relationship) + wrapperQuery = standardWrap( + `json_group_array(json_object(${fieldList}))` + ) + break + case SqlClient.POSTGRES: + wrapperQuery = standardWrap( + `json_agg(json_build_object(${fieldList}))` + ) + break + case SqlClient.MY_SQL: + case SqlClient.ORACLE: + wrapperQuery = standardWrap( + `json_arrayagg(json_object(${fieldList}))` + ) + break + case SqlClient.MS_SQL: + wrapperQuery = knex.raw( + `(SELECT ${this.quote(toAlias)} = (${knex + .select(`${fromAlias}.*`) + // @ts-ignore - from alias syntax not TS supported + .from({ + [fromAlias]: subQuery, + })} FOR JSON PATH))` + ) + break + default: + throw new Error(`JSON relationships not implement for ${sqlClient}`) + } query = query.select({ [relationship.column]: wrapperQuery }) } From e30469ce16fec403b4ba1b2881842b5e913de890 Mon Sep 17 00:00:00 2001 From: mike12345567 Date: Wed, 4 Sep 2024 18:14:24 +0100 Subject: [PATCH 35/85] Getting MariaDB to work again. --- packages/backend-core/src/sql/sql.ts | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/packages/backend-core/src/sql/sql.ts b/packages/backend-core/src/sql/sql.ts index 5dd94456f4..ec8ebe0106 100644 --- a/packages/backend-core/src/sql/sql.ts +++ b/packages/backend-core/src/sql/sql.ts @@ -917,8 +917,7 @@ class InternalBuilder { // SQL Server uses TOP - which performs a little differently to the normal LIMIT syntax // it reduces the result set rather than limiting how much data it filters over const primaryKey = `${toAlias}.${toPrimary || toKey}` - let subQuery: Knex.QueryBuilder | Knex.Raw = knex - .select(`${toAlias}.*`) + let subQuery: Knex.QueryBuilder = knex .from(toTableWithSchema) .limit(getBaseLimit()) // add sorting to get consistent order @@ -951,6 +950,7 @@ class InternalBuilder { } const standardWrap = (select: string): Knex.QueryBuilder => { + subQuery = subQuery.select(`${toAlias}.*`) // @ts-ignore - the from alias syntax isn't in Knex typing return knex.select(knex.raw(select)).from({ [toAlias]: subQuery, @@ -971,6 +971,10 @@ class InternalBuilder { ) break case SqlClient.MY_SQL: + wrapperQuery = subQuery.select( + knex.raw(`json_arrayagg(json_object(${fieldList}))`) + ) + break case SqlClient.ORACLE: wrapperQuery = standardWrap( `json_arrayagg(json_object(${fieldList}))` @@ -982,7 +986,7 @@ class InternalBuilder { .select(`${fromAlias}.*`) // @ts-ignore - from alias syntax not TS supported .from({ - [fromAlias]: subQuery, + [fromAlias]: subQuery.select(`${toAlias}.*`), })} FOR JSON PATH))` ) break From 7cdf8137c52d42419af2db13a344e6c256c7d9d5 Mon Sep 17 00:00:00 2001 From: mike12345567 Date: Thu, 5 Sep 2024 17:57:15 +0100 Subject: [PATCH 36/85] Fixing aliasing test cases. --- .../src/integrations/tests/sqlAlias.spec.ts | 91 ++++++++++--------- 1 file changed, 48 insertions(+), 43 deletions(-) diff --git a/packages/server/src/integrations/tests/sqlAlias.spec.ts b/packages/server/src/integrations/tests/sqlAlias.spec.ts index 6f34f4eb89..1ba37018dc 100644 --- a/packages/server/src/integrations/tests/sqlAlias.spec.ts +++ b/packages/server/src/integrations/tests/sqlAlias.spec.ts @@ -32,8 +32,8 @@ function multiline(sql: string) { } describe("Captures of real examples", () => { - const limit = 5000 - const relationshipLimit = 100 + const baseLimit = 5000 + const primaryLimit = 100 function getJson(name: string): QueryJson { return require(join(__dirname, "sqlQueryJson", name)) as QueryJson @@ -42,7 +42,7 @@ describe("Captures of real examples", () => { describe("create", () => { it("should create a row with relationships", () => { const queryJson = getJson("createWithRelationships.json") - let query = new Sql(SqlClient.POSTGRES, limit)._query(queryJson) + let query = new Sql(SqlClient.POSTGRES, baseLimit)._query(queryJson) expect(query).toEqual({ bindings: ["A Street", 34, "London", "A", "B", "designer", 1990], sql: multiline(`insert into "persons" ("address", "age", "city", "firstname", "lastname", "type", "year") @@ -54,40 +54,42 @@ describe("Captures of real examples", () => { describe("read", () => { it("should handle basic retrieval with relationships", () => { const queryJson = getJson("basicFetchWithRelationships.json") - let query = new Sql(SqlClient.POSTGRES, limit)._query(queryJson) + let query = new Sql(SqlClient.POSTGRES, baseLimit)._query(queryJson) expect(query).toEqual({ - bindings: [relationshipLimit, limit], + bindings: [baseLimit, baseLimit, primaryLimit], sql: expect.stringContaining( - multiline(`select "a"."year" as "a.year", "a"."firstname" as "a.firstname", "a"."personid" as "a.personid", - "a"."address" as "a.address", "a"."age" as "a.age", "a"."type" as "a.type", "a"."city" as "a.city", - "a"."lastname" as "a.lastname", "b"."executorid" as "b.executorid", "b"."taskname" as "b.taskname", - "b"."taskid" as "b.taskid", "b"."completed" as "b.completed", "b"."qaid" as "b.qaid", - "b"."executorid" as "b.executorid", "b"."taskname" as "b.taskname", "b"."taskid" as "b.taskid", - "b"."completed" as "b.completed", "b"."qaid" as "b.qaid"`) + multiline( + `select json_agg(json_build_object('executorid',"b"."executorid",'taskname',"b"."taskname",'taskid',"b"."taskid",'completed',"b"."completed",'qaid',"b"."qaid",'executorid',"b"."executorid",'taskname',"b"."taskname",'taskid',"b"."taskid",'completed',"b"."completed",'qaid',"b"."qaid")` + ) ), }) }) it("should handle filtering by relationship", () => { const queryJson = getJson("filterByRelationship.json") - let query = new Sql(SqlClient.POSTGRES, limit)._query(queryJson) + let query = new Sql(SqlClient.POSTGRES, baseLimit)._query(queryJson) expect(query).toEqual({ - bindings: [relationshipLimit, "assembling", limit], + bindings: [baseLimit, "assembling", primaryLimit], sql: expect.stringContaining( - multiline(`where COALESCE("b"."taskname" = $2, FALSE) - order by "a"."productname" asc nulls first, "a"."productid" asc limit $3`) + multiline( + `where exists (select 1 from "tasks" as "b" inner join "products_tasks" as "c" on "b"."taskid" = "c"."taskid" + where "c"."productid" = "a"."productid" and COALESCE("b"."taskname" = $2, FALSE)` + ) ), }) }) it("should handle fetching many to many relationships", () => { const queryJson = getJson("fetchManyToMany.json") - let query = new Sql(SqlClient.POSTGRES, limit)._query(queryJson) + let query = new Sql(SqlClient.POSTGRES, baseLimit)._query(queryJson) expect(query).toEqual({ - bindings: [relationshipLimit, limit], + bindings: [baseLimit, primaryLimit], sql: expect.stringContaining( - multiline(`left join "products_tasks" as "c" on "a"."productid" = "c"."productid" - left join "tasks" as "b" on "b"."taskid" = "c"."taskid" `) + multiline( + `select json_agg(json_build_object('executorid',"b"."executorid",'taskname',"b"."taskname",'taskid',"b"."taskid",'completed',"b"."completed",'qaid',"b"."qaid")) + from (select "b".* from "tasks" as "b" inner join "products_tasks" as "c" on "b"."taskid" = "c"."taskid" + where "c"."productid" = "a"."productid" order by "b"."taskid" asc limit $1` + ) ), }) }) @@ -95,22 +97,21 @@ describe("Captures of real examples", () => { it("should handle enrichment of rows", () => { const queryJson = getJson("enrichRelationship.json") const filters = queryJson.filters?.oneOf?.taskid as number[] - let query = new Sql(SqlClient.POSTGRES, limit)._query(queryJson) + let query = new Sql(SqlClient.POSTGRES, baseLimit)._query(queryJson) expect(query).toEqual({ - bindings: [...filters, limit, ...filters, limit], + bindings: [baseLimit, ...filters, baseLimit], sql: multiline( - `select "a"."executorid" as "a.executorid", "a"."taskname" as "a.taskname", "a"."taskid" as "a.taskid", - "a"."completed" as "a.completed", "a"."qaid" as "a.qaid", "b"."productname" as "b.productname", "b"."productid" as "b.productid" - from (select * from "tasks" as "a" where "a"."taskid" in ($1, $2) order by "a"."taskid" asc limit $3) as "a" - left join "products_tasks" as "c" on "a"."taskid" = "c"."taskid" left join "products" as "b" on "b"."productid" = "c"."productid" - where "a"."taskid" in ($4, $5) order by "a"."taskid" asc limit $6` + `select "a".*, (select json_agg(json_build_object('productname',"b"."productname",'productid',"b"."productid")) + from (select "b".* from "products" as "b" inner join "products_tasks" as "c" on "b"."productid" = "c"."productid" + where "c"."taskid" = "a"."taskid" order by "b"."productid" asc limit $1) as "b") as "products" + from "tasks" as "a" where "a"."taskid" in ($2, $3) order by "a"."taskid" asc limit $4` ), }) }) it("should manage query with many relationship filters", () => { const queryJson = getJson("manyRelationshipFilters.json") - let query = new Sql(SqlClient.POSTGRES, limit)._query(queryJson) + let query = new Sql(SqlClient.POSTGRES, baseLimit)._query(queryJson) const filters = queryJson.filters const notEqualsValue = Object.values(filters?.notEqual!)[0] const rangeValue: { high?: string | number; low?: string | number } = @@ -119,17 +120,18 @@ describe("Captures of real examples", () => { expect(query).toEqual({ bindings: [ - notEqualsValue, - relationshipLimit, + baseLimit, + baseLimit, + baseLimit, rangeValue.low, rangeValue.high, equalValue, - true, - limit, + notEqualsValue, + primaryLimit, ], sql: expect.stringContaining( multiline( - `where "c"."year" between $3 and $4 and COALESCE("b"."productname" = $5, FALSE)` + `where exists (select 1 from "persons" as "c" where "c"."personid" = "a"."executorid" and "c"."year" between $4 and $5)` ) ), }) @@ -139,17 +141,19 @@ describe("Captures of real examples", () => { describe("update", () => { it("should handle performing a simple update", () => { const queryJson = getJson("updateSimple.json") - let query = new Sql(SqlClient.POSTGRES, limit)._query(queryJson) + let query = new Sql(SqlClient.POSTGRES, baseLimit)._query(queryJson) expect(query).toEqual({ bindings: [1990, "C", "A Street", 34, "designer", "London", "B", 5], - sql: multiline(`update "persons" as "a" set "year" = $1, "firstname" = $2, "address" = $3, "age" = $4, - "type" = $5, "city" = $6, "lastname" = $7 where COALESCE("a"."personid" = $8, FALSE) returning *`), + sql: multiline( + `update "persons" as "a" set "year" = $1, "firstname" = $2, "address" = $3, "age" = $4, + "type" = $5, "city" = $6, "lastname" = $7 where COALESCE("a"."personid" = $8, FALSE) returning *` + ), }) }) it("should handle performing an update of relationships", () => { const queryJson = getJson("updateRelationship.json") - let query = new Sql(SqlClient.POSTGRES, limit)._query(queryJson) + let query = new Sql(SqlClient.POSTGRES, baseLimit)._query(queryJson) expect(query).toEqual({ bindings: [1990, "C", "A Street", 34, "designer", "London", "B", 5], sql: multiline(`update "persons" as "a" set "year" = $1, "firstname" = $2, "address" = $3, "age" = $4, @@ -161,12 +165,12 @@ describe("Captures of real examples", () => { describe("delete", () => { it("should handle deleting with relationships", () => { const queryJson = getJson("deleteSimple.json") - let query = new Sql(SqlClient.POSTGRES, limit)._query(queryJson) + let query = new Sql(SqlClient.POSTGRES, baseLimit)._query(queryJson) expect(query).toEqual({ bindings: ["ddd", ""], sql: multiline(`delete from "compositetable" as "a" where COALESCE("a"."keypartone" = $1, FALSE) and COALESCE("a"."keyparttwo" = $2, FALSE) - returning "a"."keyparttwo" as "a.keyparttwo", "a"."keypartone" as "a.keypartone", "a"."name" as "a.name"`), + returning "a".*`), }) }) }) @@ -174,7 +178,7 @@ describe("Captures of real examples", () => { describe("returning (everything bar Postgres)", () => { it("should be able to handle row returning", () => { const queryJson = getJson("createSimple.json") - const SQL = new Sql(SqlClient.MS_SQL, limit) + const SQL = new Sql(SqlClient.MS_SQL, baseLimit) let query = SQL._query(queryJson, { disableReturning: true }) expect(query).toEqual({ sql: "insert into [people] ([age], [name]) values (@p0, @p1)", @@ -187,10 +191,11 @@ describe("Captures of real examples", () => { returningQuery = input }, queryJson) expect(returningQuery).toEqual({ - sql: multiline(`select top (@p0) * from (select top (@p1) * from [people] where CASE WHEN [people].[name] = @p2 - THEN 1 ELSE 0 END = 1 and CASE WHEN [people].[age] = @p3 THEN 1 ELSE 0 END = 1 order by [people].[name] asc) as [people] - where CASE WHEN [people].[name] = @p4 THEN 1 ELSE 0 END = 1 and CASE WHEN [people].[age] = @p5 THEN 1 ELSE 0 END = 1`), - bindings: [5000, 1, "Test", 22, "Test", 22], + sql: multiline( + `select top (@p0) * from [people] where CASE WHEN [people].[name] = @p1 THEN 1 ELSE 0 END = 1 + and CASE WHEN [people].[age] = @p2 THEN 1 ELSE 0 END = 1 order by [people].[name] asc` + ), + bindings: [1, "Test", 22], }) }) }) From 888c4214bdbef1c5202144ff1c9ca0449e07072e Mon Sep 17 00:00:00 2001 From: mike12345567 Date: Thu, 5 Sep 2024 18:12:53 +0100 Subject: [PATCH 37/85] Fixing SQL unit tests. --- .../server/src/integrations/tests/sql.spec.ts | 32 +++++++++---------- 1 file changed, 16 insertions(+), 16 deletions(-) diff --git a/packages/server/src/integrations/tests/sql.spec.ts b/packages/server/src/integrations/tests/sql.spec.ts index a6e63c434d..c434ec42cb 100644 --- a/packages/server/src/integrations/tests/sql.spec.ts +++ b/packages/server/src/integrations/tests/sql.spec.ts @@ -160,16 +160,16 @@ describe("SQL query builder", () => { it("should add the schema to the LEFT JOIN", () => { const query = sql._query(generateRelationshipJson({ schema: "production" })) expect(query).toEqual({ - bindings: [500, 5000], - sql: `select "brands"."brand_id" as "brands.brand_id", "brands"."brand_name" as "brands.brand_name", "products"."product_id" as "products.product_id", "products"."product_name" as "products.product_name", "products"."brand_id" as "products.brand_id" from (select * from "production"."brands" order by "test"."id" asc limit $1) as "brands" left join "production"."products" as "products" on "brands"."brand_id" = "products"."brand_id" order by "test"."id" asc limit $2`, + bindings: [5000, limit], + sql: `select "brands".*, (select json_agg(json_build_object('product_id',"products"."product_id",'product_name',"products"."product_name",'brand_id',"products"."brand_id")) from (select "products".* from "production"."products" as "products" where "products"."brand_id" = "brands"."brand_id" order by "products"."brand_id" asc limit $1) as "products") as "products" from "production"."brands" order by "test"."id" asc limit $2`, }) }) it("should handle if the schema is not present when doing a LEFT JOIN", () => { const query = sql._query(generateRelationshipJson()) expect(query).toEqual({ - bindings: [500, 5000], - sql: `select "brands"."brand_id" as "brands.brand_id", "brands"."brand_name" as "brands.brand_name", "products"."product_id" as "products.product_id", "products"."product_name" as "products.product_name", "products"."brand_id" as "products.brand_id" from (select * from "brands" order by "test"."id" asc limit $1) as "brands" left join "products" as "products" on "brands"."brand_id" = "products"."brand_id" order by "test"."id" asc limit $2`, + bindings: [5000, limit], + sql: `select "brands".*, (select json_agg(json_build_object('product_id',"products"."product_id",'product_name',"products"."product_name",'brand_id',"products"."brand_id")) from (select "products".* from "products" as "products" where "products"."brand_id" = "brands"."brand_id" order by "products"."brand_id" asc limit $1) as "products") as "products" from "brands" order by "test"."id" asc limit $2`, }) }) @@ -178,8 +178,8 @@ describe("SQL query builder", () => { generateManyRelationshipJson({ schema: "production" }) ) expect(query).toEqual({ - bindings: [500, 5000], - sql: `select "stores"."store_id" as "stores.store_id", "stores"."store_name" as "stores.store_name", "products"."product_id" as "products.product_id", "products"."product_name" as "products.product_name" from (select * from "production"."stores" order by "test"."id" asc limit $1) as "stores" left join "production"."stocks" as "stocks" on "stores"."store_id" = "stocks"."store_id" left join "production"."products" as "products" on "products"."product_id" = "stocks"."product_id" order by "test"."id" asc limit $2`, + bindings: [5000, limit], + sql: `select "stores".*, (select json_agg(json_build_object('product_id',"products"."product_id",'product_name',"products"."product_name")) from (select "products".* from "production"."products" as "products" inner join "production"."stocks" as "stocks" on "products"."product_id" = "stocks"."product_id" where "stocks"."store_id" = "stores"."store_id" order by "products"."product_id" asc limit $1) as "products") as "products" from "production"."stores" order by "test"."id" asc limit $2`, }) }) @@ -194,8 +194,8 @@ describe("SQL query builder", () => { }) ) expect(query).toEqual({ - bindings: ["john%", limit, "john%", 5000], - sql: `select * from (select * from (select * from (select * from "test" where LOWER("test"."name") LIKE :1 order by "test"."id" asc) where rownum <= :2) "test" where LOWER("test"."name") LIKE :3 order by "test"."id" asc) where rownum <= :4`, + bindings: ["john%", limit], + sql: `select * from (select * from "test" where LOWER("test"."name") LIKE :1 order by "test"."id" asc) where rownum <= :2`, }) query = new Sql(SqlClient.ORACLE, limit)._query( @@ -210,8 +210,8 @@ describe("SQL query builder", () => { ) const filterSet = [`%20%`, `%25%`, `%"john"%`, `%"mary"%`] expect(query).toEqual({ - bindings: [...filterSet, limit, ...filterSet, 5000], - sql: `select * from (select * from (select * from (select * from "test" where COALESCE(LOWER("test"."age"), '') LIKE :1 AND COALESCE(LOWER("test"."age"), '') LIKE :2 and COALESCE(LOWER("test"."name"), '') LIKE :3 AND COALESCE(LOWER("test"."name"), '') LIKE :4 order by "test"."id" asc) where rownum <= :5) "test" where COALESCE(LOWER("test"."age"), '') LIKE :6 AND COALESCE(LOWER("test"."age"), '') LIKE :7 and COALESCE(LOWER("test"."name"), '') LIKE :8 AND COALESCE(LOWER("test"."name"), '') LIKE :9 order by "test"."id" asc) where rownum <= :10`, + bindings: [...filterSet, limit], + sql: `select * from (select * from "test" where COALESCE(LOWER("test"."age"), '') LIKE :1 AND COALESCE(LOWER("test"."age"), '') LIKE :2 and COALESCE(LOWER("test"."name"), '') LIKE :3 AND COALESCE(LOWER("test"."name"), '') LIKE :4 order by "test"."id" asc) where rownum <= :5`, }) query = new Sql(SqlClient.ORACLE, limit)._query( @@ -224,8 +224,8 @@ describe("SQL query builder", () => { }) ) expect(query).toEqual({ - bindings: [`%jo%`, limit, `%jo%`, 5000], - sql: `select * from (select * from (select * from (select * from "test" where LOWER("test"."name") LIKE :1 order by "test"."id" asc) where rownum <= :2) "test" where LOWER("test"."name") LIKE :3 order by "test"."id" asc) where rownum <= :4`, + bindings: [`%jo%`, limit], + sql: `select * from (select * from "test" where LOWER("test"."name") LIKE :1 order by "test"."id" asc) where rownum <= :2`, }) }) @@ -242,8 +242,8 @@ describe("SQL query builder", () => { ) expect(query).toEqual({ - bindings: ["John", limit, "John", 5000], - sql: `select * from (select * from (select * from (select * from "test" where (to_char("test"."name") IS NOT NULL AND to_char("test"."name") = :1) order by "test"."id" asc) where rownum <= :2) "test" where (to_char("test"."name") IS NOT NULL AND to_char("test"."name") = :3) order by "test"."id" asc) where rownum <= :4`, + bindings: ["John", limit], + sql: `select * from (select * from "test" where (to_char("test"."name") IS NOT NULL AND to_char("test"."name") = :1) order by "test"."id" asc) where rownum <= :2`, }) }) @@ -260,8 +260,8 @@ describe("SQL query builder", () => { ) expect(query).toEqual({ - bindings: ["John", limit, "John", 5000], - sql: `select * from (select * from (select * from (select * from "test" where (to_char("test"."name") IS NOT NULL AND to_char("test"."name") != :1) OR to_char("test"."name") IS NULL order by "test"."id" asc) where rownum <= :2) "test" where (to_char("test"."name") IS NOT NULL AND to_char("test"."name") != :3) OR to_char("test"."name") IS NULL order by "test"."id" asc) where rownum <= :4`, + bindings: ["John", limit], + sql: `select * from (select * from "test" where (to_char("test"."name") IS NOT NULL AND to_char("test"."name") != :1) OR to_char("test"."name") IS NULL order by "test"."id" asc) where rownum <= :2`, }) }) }) From f7d9b8a9b31da5ee8357e42398c5bf733d738e9c Mon Sep 17 00:00:00 2001 From: mike12345567 Date: Thu, 5 Sep 2024 19:04:45 +0100 Subject: [PATCH 38/85] Updating select statement generation. --- packages/backend-core/src/sql/sql.ts | 142 ++++++++++++++------------- 1 file changed, 72 insertions(+), 70 deletions(-) diff --git a/packages/backend-core/src/sql/sql.ts b/packages/backend-core/src/sql/sql.ts index ec8ebe0106..b1a2bc060a 100644 --- a/packages/backend-core/src/sql/sql.ts +++ b/packages/backend-core/src/sql/sql.ts @@ -94,6 +94,23 @@ class InternalBuilder { }) } + // states the various situations in which we need a full mapped select statement + private readonly SPECIAL_SELECT_CASES = { + POSTGRES_MONEY: (field: FieldSchema | undefined) => { + return ( + this.client === SqlClient.POSTGRES && + field?.externalType?.includes("money") + ) + }, + MSSQL_DATES: (field: FieldSchema | undefined) => { + return ( + this.client === SqlClient.MS_SQL && + field?.type === FieldType.DATETIME && + field.timeOnly + ) + }, + } + get table(): Table { return this.query.meta.table } @@ -127,8 +144,20 @@ class InternalBuilder { .join(".") } + private isFullSelectStatementRequired(): boolean { + const { meta } = this.query + for (let column of Object.values(meta.table.schema)) { + if (this.SPECIAL_SELECT_CASES.POSTGRES_MONEY(column)) { + return true + } else if (this.SPECIAL_SELECT_CASES.MSSQL_DATES(column)) { + return true + } + } + return false + } + private generateSelectStatement(): (string | Knex.Raw)[] | "*" { - const { endpoint, resource, tableAliases } = this.query + const { meta, endpoint, resource, tableAliases } = this.query if (!resource || !resource.fields || resource.fields.length === 0) { return "*" @@ -137,75 +166,48 @@ class InternalBuilder { const alias = tableAliases?.[endpoint.entityId] ? tableAliases?.[endpoint.entityId] : endpoint.entityId - return [this.knex.raw(`${this.quote(alias)}.*`)] - // - // - // const schema = meta.table.schema - // return resource.fields.map(field => { - // const parts = field.split(/\./g) - // let table: string | undefined = undefined - // let column: string | undefined = undefined - // - // // Just a column name, e.g.: "column" - // if (parts.length === 1) { - // column = parts[0] - // } - // - // // A table name and a column name, e.g.: "table.column" - // if (parts.length === 2) { - // table = parts[0] - // column = parts[1] - // } - // - // // A link doc, e.g.: "table.doc1.fieldName" - // if (parts.length > 2) { - // table = parts[0] - // column = parts.slice(1).join(".") - // } - // - // if (!column) { - // throw new Error(`Invalid field name: ${field}`) - // } - // - // const columnSchema = schema[column] - // - // if ( - // this.client === SqlClient.POSTGRES && - // columnSchema?.externalType?.includes("money") - // ) { - // return this.knex.raw( - // `${this.quotedIdentifier( - // [table, column].join(".") - // )}::money::numeric as ${this.quote(field)}` - // ) - // } - // - // if ( - // this.client === SqlClient.MS_SQL && - // columnSchema?.type === FieldType.DATETIME && - // columnSchema.timeOnly - // ) { - // // Time gets returned as timestamp from mssql, not matching the expected - // // HH:mm format - // return this.knex.raw(`CONVERT(varchar, ${field}, 108) as "${field}"`) - // } - // - // // There's at least two edge cases being handled in the expression below. - // // 1. The column name could start/end with a space, and in that case we - // // want to preseve that space. - // // 2. Almost all column names are specified in the form table.column, except - // // in the case of relationships, where it's table.doc1.column. In that - // // case, we want to split it into `table`.`doc1.column` for reasons that - // // aren't actually clear to me, but `table`.`doc1` breaks things with the - // // sample data tests. - // if (table) { - // return this.knex.raw( - // `${this.quote(table)}.${this.quote(column)} as ${this.quote(field)}` - // ) - // } else { - // return this.knex.raw(`${this.quote(field)} as ${this.quote(field)}`) - // } - // }) + const schema = meta.table.schema + if (!this.isFullSelectStatementRequired()) { + return [this.knex.raw(`${this.quote(alias)}.*`)] + } + // get just the fields for this table + return resource.fields + .map(field => { + const parts = field.split(/\./g) + let table: string | undefined = undefined + let column = parts[0] + + // Just a column name, e.g.: "column" + if (parts.length > 1) { + table = parts[0] + column = parts.slice(1).join(".") + } + + return { table, column, field } + }) + .filter(({ table }) => !table || table === alias) + .map(({ table, column, field }) => { + const columnSchema = schema[column] + + if (this.SPECIAL_SELECT_CASES.POSTGRES_MONEY(columnSchema)) { + return this.knex.raw( + `${this.quotedIdentifier( + [table, column].join(".") + )}::money::numeric as ${this.quote(field)}` + ) + } + + if (this.SPECIAL_SELECT_CASES.MSSQL_DATES(columnSchema)) { + // Time gets returned as timestamp from mssql, not matching the expected + // HH:mm format + return this.knex.raw(`CONVERT(varchar, ${field}, 108) as "${field}"`) + } + + const quoted = table + ? `${this.quote(table)}.${this.quote(column)}` + : this.quote(field) + return this.knex.raw(quoted) + }) } // OracleDB can't use character-large-objects (CLOBs) in WHERE clauses, From cdb0e2509c2067db0dc8a9f84e3e775c2b76a2ae Mon Sep 17 00:00:00 2001 From: Martin McKeaveney Date: Fri, 6 Sep 2024 14:53:26 +0100 Subject: [PATCH 39/85] testing configuration, unit tests for AI config --- .../core/utilities/structures/licenses.ts | 10 + packages/builder/package.json | 1 + .../portal/settings/ai/AISettings.spec.js | 96 ++++ .../builder/portal/settings/ai/ai.spec.ts | 0 .../builder/portal/settings/ai/index.svelte | 5 +- .../{vite.config.js => vite.config.mjs} | 4 +- packages/builder/vitest.setup.js | 5 +- yarn.lock | 476 +++++++++--------- 8 files changed, 357 insertions(+), 240 deletions(-) create mode 100644 packages/builder/src/pages/builder/portal/settings/ai/AISettings.spec.js delete mode 100644 packages/builder/src/pages/builder/portal/settings/ai/ai.spec.ts rename packages/builder/{vite.config.js => vite.config.mjs} (98%) diff --git a/packages/backend-core/tests/core/utilities/structures/licenses.ts b/packages/backend-core/tests/core/utilities/structures/licenses.ts index bb452f9ad5..cf422637d6 100644 --- a/packages/backend-core/tests/core/utilities/structures/licenses.ts +++ b/packages/backend-core/tests/core/utilities/structures/licenses.ts @@ -55,6 +55,11 @@ export function quotas(): Quotas { value: 1, triggers: [], }, + budibaseAICredits: { + name: "Budibase AI Credits", + value: 1, + triggers: [], + } }, static: { rows: { @@ -87,6 +92,11 @@ export function quotas(): Quotas { value: 1, triggers: [], }, + aiCustomConfigs: { + name: "Plugins", + value: 1, + triggers: [], + }, }, }, constant: { diff --git a/packages/builder/package.json b/packages/builder/package.json index f44c2ea549..f9e6becbab 100644 --- a/packages/builder/package.json +++ b/packages/builder/package.json @@ -93,6 +93,7 @@ "identity-obj-proxy": "^3.0.0", "jest": "29.7.0", "jsdom": "^21.1.1", + "resize-observer-polyfill": "^1.5.1", "svelte-jester": "^1.3.2", "vite": "^4.5.0", "vite-plugin-static-copy": "^0.17.0", diff --git a/packages/builder/src/pages/builder/portal/settings/ai/AISettings.spec.js b/packages/builder/src/pages/builder/portal/settings/ai/AISettings.spec.js new file mode 100644 index 0000000000..5ca14c7898 --- /dev/null +++ b/packages/builder/src/pages/builder/portal/settings/ai/AISettings.spec.js @@ -0,0 +1,96 @@ +import { it, expect, describe, vi } from "vitest" +import AISettings from "./index.svelte" +import { render } from "@testing-library/svelte" +import { admin, licensing } from "stores/portal" + +vi.spyOn(notifications, "error").mockImplementation(vi.fn) +vi.spyOn(notifications, "success").mockImplementation(vi.fn) + +const Hosting = { + Cloud: "cloud", + Self: "self", +} + +function setupEnv( + hosting, + features = {}) { + const defaultFeatures = { + budibaseAIEnabled: false, + customAIConfigsEnabled: false, + ...features + } + admin.subscribe = vi.fn().mockImplementation(callback => { + callback({cloud: hosting === Hosting.Cloud}) + return () => { + } + }) + licensing.subscribe = vi.fn().mockImplementation(callback => { + callback(defaultFeatures) + return () => { + } + }) +} + +describe("AISettings", () => { + let instance = null + + afterEach(() => { + vi.restoreAllMocks() + }) + + it("that the AISettings is rendered", () => { + instance = render(AISettings, {}) + expect(instance).toBeDefined() + }) + + describe("Licensing", () => { + it("should show the premium label on self host for custom configs", async () => { + setupEnv(Hosting.Self) + instance = render(AISettings, {}) + const premiumTag = instance.queryByText("Premium") + expect(premiumTag).toBeInTheDocument() + }) + + it("should show the enterprise label on cloud for custom configs", async () => { + setupEnv(Hosting.Cloud) + instance = render(AISettings, {}) + const enterpriseTag = instance.queryByText("Enterprise") + expect(enterpriseTag).toBeInTheDocument() + }) + + it("should show the premium label on cloud when Budibase AI isn't enabled", async () => { + setupEnv(Hosting.Cloud) + instance = render(AISettings, {}) + const premiumTag = instance.queryByText("Premium") + expect(premiumTag).toBeInTheDocument() + }) + + it("should not show the add configuration button if the user doesn't have the correct license on cloud", async () => { + let addConfigurationButton + + setupEnv(Hosting.Cloud) + instance = render(AISettings) + addConfigurationButton = instance.queryByText("Add configuration") + expect(addConfigurationButton).not.toBeInTheDocument() + + setupEnv(Hosting.Cloud, {customAIConfigsEnabled: true}) + instance = render(AISettings) + addConfigurationButton = instance.queryByText("Add configuration") + expect(addConfigurationButton).toBeInTheDocument() + }) + + it("should not show the add configuration button if the user doesn't have the correct license on self host", async () => { + let addConfigurationButton + + setupEnv(Hosting.Self) + instance = render(AISettings) + addConfigurationButton = instance.queryByText("Add configuration") + expect(addConfigurationButton).not.toBeInTheDocument() + + setupEnv(Hosting.Self, {customAIConfigsEnabled: true}) + instance = render(AISettings, {}) + addConfigurationButton = instance.queryByText("Add configuration") + expect(addConfigurationButton).toBeInTheDocument() + }) + }) +}) diff --git a/packages/builder/src/pages/builder/portal/settings/ai/ai.spec.ts b/packages/builder/src/pages/builder/portal/settings/ai/ai.spec.ts deleted file mode 100644 index e69de29bb2..0000000000 diff --git a/packages/builder/src/pages/builder/portal/settings/ai/index.svelte b/packages/builder/src/pages/builder/portal/settings/ai/index.svelte index 83b24a5b97..dc7853b398 100644 --- a/packages/builder/src/pages/builder/portal/settings/ai/index.svelte +++ b/packages/builder/src/pages/builder/portal/settings/ai/index.svelte @@ -32,7 +32,6 @@ async function fetchAIConfig() { try { - // Fetch the AI configs const aiDoc = await API.getConfig(ConfigTypes.AI) if (aiDoc._id) { fullAIConfig = aiDoc @@ -105,8 +104,8 @@ modal.show() } - onMount(async () => { - await fetchAIConfig() + onMount(() => { + fetchAIConfig() }) diff --git a/packages/builder/vite.config.js b/packages/builder/vite.config.mjs similarity index 98% rename from packages/builder/vite.config.js rename to packages/builder/vite.config.mjs index 995cd36adb..a84fba2dae 100644 --- a/packages/builder/vite.config.js +++ b/packages/builder/vite.config.mjs @@ -35,7 +35,7 @@ export default defineConfig(({ mode }) => { // Copy fonts to an additional path so that svelte's automatic // prefixing of the base URL path can still resolve assets copyFonts("builder/fonts"), - ] +] return { test: { @@ -79,7 +79,7 @@ export default defineConfig(({ mode }) => { "process.env.POSTHOG_TOKEN": JSON.stringify(process.env.POSTHOG_TOKEN), }), copyFonts("fonts"), - ...(isProduction ? [] : devOnlyPlugins), + ...(isProduction ? [] : devOnlyPlugins), ], optimizeDeps: { exclude: ["@roxi/routify", "fsevents"], diff --git a/packages/builder/vitest.setup.js b/packages/builder/vitest.setup.js index 6581df83e1..92df5e9600 100644 --- a/packages/builder/vitest.setup.js +++ b/packages/builder/vitest.setup.js @@ -1,4 +1,7 @@ -import { expect } from "vitest" +import { expect, vi } from "vitest" +import "@testing-library/jest-dom/vitest" + +global.ResizeObserver = require("resize-observer-polyfill") expect.extend({ toBeFunc: received => { diff --git a/yarn.lock b/yarn.lock index 655d7d78fc..4e4e528334 100644 --- a/yarn.lock +++ b/yarn.lock @@ -8,9 +8,9 @@ integrity sha512-1Yjs2SvM8TflER/OD3cOjhWWOZb58A2t7wpE2S9XfBYTiIl+XFhQG2bjy4Pu1I+EAlCNUzRDYDdFwFYUKvXcIA== "@adobe/css-tools@^4.3.2": - version "4.3.3" - resolved "https://registry.yarnpkg.com/@adobe/css-tools/-/css-tools-4.3.3.tgz#90749bde8b89cd41764224f5aac29cd4138f75ff" - integrity sha512-rE0Pygv0sEZ4vBWHlAgJLGDU7Pm8xoO6p3wsEceb7GYAjScrOHpEo8KK/eVkAcnSM+slAEtXjA2JpdjLp4fJQQ== + version "4.4.0" + resolved "https://registry.yarnpkg.com/@adobe/css-tools/-/css-tools-4.4.0.tgz#728c484f4e10df03d5a3acd0d8adcbbebff8ad63" + integrity sha512-Ff9+ksdQQB3rMncgqDK78uLznstjyfIf2Arnh22pW8kBpLs6rpKDwgnZT46hin5Hl1WzazzK64DOrhSwYpS7bQ== "@adobe/spectrum-css-workflow-icons@1.2.1": version "1.2.1" @@ -1995,14 +1995,14 @@ resolved "https://registry.yarnpkg.com/@babel/regjsgen/-/regjsgen-0.8.0.tgz#f0ba69b075e1f05fb2825b7fad991e7adbb18310" integrity sha512-x/rqGMdzj+fWZvCOYForTghzbtqPDZ5gPwaoNGHdgDfF2QA/XZbCBp4Moo5scrkAMPhB7z26XM/AaHuIJdgauA== -"@babel/runtime@^7.10.5", "@babel/runtime@^7.12.5", "@babel/runtime@^7.15.4", "@babel/runtime@^7.8.4", "@babel/runtime@^7.9.2": +"@babel/runtime@^7.10.5", "@babel/runtime@^7.15.4", "@babel/runtime@^7.8.4": version "7.24.7" resolved "https://registry.yarnpkg.com/@babel/runtime/-/runtime-7.24.7.tgz#f4f0d5530e8dbdf59b3451b9b3e594b6ba082e12" integrity sha512-UwgBRMjJP+xv857DCngvqXI3Iq6J4v0wXmwc6sapg+zyhbwmQX67LUEFrkK5tbyJ30jGuG3ZvWpBiB9LCy1kWw== dependencies: regenerator-runtime "^0.14.0" -"@babel/runtime@^7.13.10": +"@babel/runtime@^7.12.5", "@babel/runtime@^7.13.10", "@babel/runtime@^7.9.2": version "7.25.6" resolved "https://registry.yarnpkg.com/@babel/runtime/-/runtime-7.25.6.tgz#9afc3289f7184d8d7f98b099884c26317b9264d2" integrity sha512-VBj9MYyDb9tuLq7yzqjgzt6Q+IBQLrGZfdjOekyEirZPHxXWoTSGUTMrpsfi58Up73d13NfYLv8HT9vmznjzhQ== @@ -2053,44 +2053,6 @@ resolved "https://registry.yarnpkg.com/@bcoe/v8-coverage/-/v8-coverage-0.2.3.tgz#75a2e8b51cb758a7553d6804a5932d7aace75c39" integrity sha512-0hYQ8SB4Db5zvZB4axdMHGwEaQjkZzFjQiN9LVYvIFB2nSUHW9tYpxWriPrWDASIxiaXax83REcLxuSdnGPZtw== -"@budibase/backend-core@2.29.24": - version "0.0.0" - dependencies: - "@budibase/nano" "10.1.5" - "@budibase/pouchdb-replication-stream" "1.2.11" - "@budibase/shared-core" "0.0.0" - "@budibase/types" "0.0.0" - aws-cloudfront-sign "3.0.2" - aws-sdk "2.1030.0" - bcrypt "5.1.0" - bcryptjs "2.4.3" - bull "4.10.1" - correlation-id "4.0.0" - dd-trace "5.2.0" - dotenv "16.0.1" - ioredis "5.3.2" - joi "17.6.0" - jsonwebtoken "9.0.2" - knex "2.4.2" - koa-passport "^6.0.0" - koa-pino-logger "4.0.0" - lodash "4.17.21" - node-fetch "2.6.7" - passport-google-oauth "2.0.0" - passport-local "1.0.0" - passport-oauth2-refresh "^2.1.0" - pino "8.11.0" - pino-http "8.3.3" - posthog-node "4.0.1" - pouchdb "7.3.0" - pouchdb-find "7.2.2" - redlock "4.2.0" - rotating-file-stream "3.1.0" - sanitize-s3-objectkey "0.0.1" - semver "^7.5.4" - tar-fs "2.1.1" - uuid "^8.3.2" - "@budibase/handlebars-helpers@^0.13.2": version "0.13.2" resolved "https://registry.yarnpkg.com/@budibase/handlebars-helpers/-/handlebars-helpers-0.13.2.tgz#73ab51c464e91fd955b429017648e0257060db77" @@ -2133,44 +2095,6 @@ pouchdb-promise "^6.0.4" through2 "^2.0.0" -"@budibase/pro@npm:@budibase/pro@latest": - version "2.29.24" - resolved "https://registry.yarnpkg.com/@budibase/pro/-/pro-2.29.24.tgz#2dbd4c6c0f757aab7e17c413c6d6e4520086f9ac" - integrity sha512-m1v24UD6O21Vbrfsuo5kC5oeg7FzjWO2w8TQMw1VvPKmdIqqclaKDPTPytxwllTMkapMDRNzM5cQzqnQ3yHf6A== - dependencies: - "@budibase/backend-core" "2.29.24" - "@budibase/shared-core" "2.29.24" - "@budibase/string-templates" "2.29.24" - "@budibase/types" "2.29.24" - "@koa/router" "8.0.8" - bull "4.10.1" - joi "17.6.0" - jsonwebtoken "9.0.2" - lru-cache "^7.14.1" - memorystream "^0.3.1" - node-fetch "2.6.7" - scim-patch "^0.8.1" - scim2-parse-filter "^0.2.8" - -"@budibase/shared-core@2.29.24": - version "0.0.0" - dependencies: - "@budibase/types" "0.0.0" - cron-validate "1.4.5" - -"@budibase/string-templates@2.29.24": - version "0.0.0" - dependencies: - "@budibase/handlebars-helpers" "^0.13.2" - dayjs "^1.10.8" - handlebars "^4.7.8" - lodash.clonedeep "^4.5.0" - -"@budibase/types@2.29.24": - version "0.0.0" - dependencies: - scim-patch "^0.8.1" - "@bull-board/api@5.10.2": version "5.10.2" resolved "https://registry.yarnpkg.com/@bull-board/api/-/api-5.10.2.tgz#ae8ff6918b23897bf879a6ead3683f964374c4b3" @@ -5338,9 +5262,9 @@ redent "^3.0.0" "@testing-library/svelte@^4.1.0": - version "4.1.0" - resolved "https://registry.yarnpkg.com/@testing-library/svelte/-/svelte-4.1.0.tgz#de6fa34d13d99505e68134ef2acfbafdc03ed39a" - integrity sha512-MJqe7x9WowkiAVdk9mvazEC2ktFZdmK2OqFVoO557PC37aBemQ4ozqdK3yrG34Zg9kuln3qgTVeLSh08e69AMw== + version "4.2.3" + resolved "https://registry.yarnpkg.com/@testing-library/svelte/-/svelte-4.2.3.tgz#bd0fd3dde5c63012da95d58ee640c0b44fd27a04" + integrity sha512-8vM2+JSPc6wZWkO9ICPmHvzacjy8jBw+iVjmNs+0VsPV3AO3v4P8qCLWTaQ9nYW/e+IR1BCy3MM3Uqg21dlBkw== dependencies: "@testing-library/dom" "^9.3.1" @@ -5412,9 +5336,9 @@ "@types/readdir-glob" "*" "@types/aria-query@^5.0.1": - version "5.0.1" - resolved "https://registry.yarnpkg.com/@types/aria-query/-/aria-query-5.0.1.tgz#3286741fb8f1e1580ac28784add4c7a1d49bdfbc" - integrity sha512-XTIieEY+gvJ39ChLcB4If5zHtPxt3Syj5rgZR+e1ctpmK8NjPf0zFqsz4JpLJT0xla9GFDKjy8Cpu331nrmE1Q== + version "5.0.4" + resolved "https://registry.yarnpkg.com/@types/aria-query/-/aria-query-5.0.4.tgz#1a31c3d378850d2778dabb6374d036dcba4ba708" + integrity sha512-rfT93uj5s0PRL7EzccGMs3brplhcrghnDoV26NqKhCAS1hVo+WdNsPvE/yb6ilfr5hi2MEk6d5EWJTKdxg8jVw== "@types/babel__core@^7.1.14": version "7.20.0" @@ -5485,9 +5409,9 @@ "@types/chai" "*" "@types/chai@*", "@types/chai@^4.3.4": - version "4.3.16" - resolved "https://registry.yarnpkg.com/@types/chai/-/chai-4.3.16.tgz#b1572967f0b8b60bf3f87fe1d854a5604ea70c82" - integrity sha512-PatH4iOdyh3MyWtmHVFXLWCCIhUbopaltqddG9BzB+gMIzee2MJrvd+jouii9Z3wzQJruGWAm7WOMjgfG8hQlQ== + version "4.3.19" + resolved "https://registry.yarnpkg.com/@types/chai/-/chai-4.3.19.tgz#14519f437361d41e84102ed3fbc922ddace3e228" + integrity sha512-2hHHvQBVE2FiSK4eN0Br6snX9MtolHaTo/batnLjlGRhoQzlCL61iVpxoqO7SfFyOw+P/pwv+0zNHzKoGWz9Cw== "@types/chance@1.1.3": version "1.1.3" @@ -6922,16 +6846,16 @@ acorn@^7.1.1: resolved "https://registry.yarnpkg.com/acorn/-/acorn-7.4.1.tgz#feaed255973d2e77555b83dbc08851a6c63520fa" integrity sha512-nQyp0o1/mNdbTO1PO6kHkwSrmgZ0MT/jCCpNiwbUjGoRN4dlBhqJtoQuCnEOKzgTVwg0ZWiCoQy6SxMebQVh8A== -acorn@^8.1.0, acorn@^8.10.0, acorn@^8.11.0, acorn@^8.11.3, acorn@^8.2.4, acorn@^8.4.1, acorn@^8.8.1, acorn@^8.8.2, acorn@^8.9.0: - version "8.12.0" - resolved "https://registry.yarnpkg.com/acorn/-/acorn-8.12.0.tgz#1627bfa2e058148036133b8d9b51a700663c294c" - integrity sha512-RTvkC4w+KNXrM39/lWCUaG0IbRkWdCv7W/IOW9oU6SawyxulvkQy5HQPVTKxEjczcUvapcrw3cFx/60VN/NRNw== - -acorn@^8.12.0, acorn@^8.12.1: +acorn@^8.1.0, acorn@^8.11.3, acorn@^8.12.0, acorn@^8.12.1, acorn@^8.8.1: version "8.12.1" resolved "https://registry.yarnpkg.com/acorn/-/acorn-8.12.1.tgz#71616bdccbe25e27a54439e0046e89ca76df2248" integrity sha512-tcpGyI9zbizT9JbV6oYE477V6mTlXvvi0T0G3SNIYE2apm/G5huBa1+K89VGeovbg+jycCrfhl3ADxErOuO6Jg== +acorn@^8.10.0, acorn@^8.11.0, acorn@^8.2.4, acorn@^8.4.1, acorn@^8.8.2, acorn@^8.9.0: + version "8.12.0" + resolved "https://registry.yarnpkg.com/acorn/-/acorn-8.12.0.tgz#1627bfa2e058148036133b8d9b51a700663c294c" + integrity sha512-RTvkC4w+KNXrM39/lWCUaG0IbRkWdCv7W/IOW9oU6SawyxulvkQy5HQPVTKxEjczcUvapcrw3cFx/60VN/NRNw== + add-stream@^1.0.0: version "1.0.0" resolved "https://registry.yarnpkg.com/add-stream/-/add-stream-1.0.0.tgz#6a7990437ca736d5e1288db92bd3266d5f5cb2aa" @@ -7259,7 +7183,7 @@ aria-query@^5.0.0, aria-query@^5.3.0: dependencies: dequal "^2.0.3" -array-buffer-byte-length@^1.0.1: +array-buffer-byte-length@^1.0.0, array-buffer-byte-length@^1.0.1: version "1.0.1" resolved "https://registry.yarnpkg.com/array-buffer-byte-length/-/array-buffer-byte-length-1.0.1.tgz#1e5583ec16763540a27ae52eed99ff899223568f" integrity sha512-ahC5W1xgou+KTXix4sAO8Ki12Q+jf4i0+tmk3sC+zgcynshkHxzpXdImBehiUYKKKDwvfFiJl1tZt6ewscS1Mg== @@ -7524,7 +7448,30 @@ axios-retry@^3.1.9: "@babel/runtime" "^7.15.4" is-retry-allowed "^2.2.0" -axios@0.24.0, axios@1.1.3, axios@1.6.3, axios@^0.21.1, axios@^1.0.0, axios@^1.1.3, axios@^1.4.0, axios@^1.5.0, axios@^1.6.2: +axios@0.24.0: + version "0.24.0" + resolved "https://registry.yarnpkg.com/axios/-/axios-0.24.0.tgz#804e6fa1e4b9c5288501dd9dff56a7a0940d20d6" + integrity sha512-Q6cWsys88HoPgAaFAVUb0WpPk0O8iTeisR9IMqy9G8AbO4NlpVknrnQS03zzF9PGAWgO3cgletO3VjV/P7VztA== + dependencies: + follow-redirects "^1.14.4" + +axios@1.1.3: + version "1.1.3" + resolved "https://registry.yarnpkg.com/axios/-/axios-1.1.3.tgz#8274250dada2edf53814ed7db644b9c2866c1e35" + integrity sha512-00tXVRwKx/FZr/IDVFt4C+f9FYairX517WoGCL6dpOntqLkZofjhu43F/Xl44UOpqa+9sLFDrG/XAnFsUYgkDA== + dependencies: + follow-redirects "^1.15.0" + form-data "^4.0.0" + proxy-from-env "^1.1.0" + +axios@^0.21.1: + version "0.21.4" + resolved "https://registry.yarnpkg.com/axios/-/axios-0.21.4.tgz#c67b90dc0568e5c1cf2b0b858c43ba28e2eda575" + integrity sha512-ut5vewkiu8jjGBdqpM44XxjuCjq9LAKeHVmoVfHVzy8eHgxxq8SbAVQNovDA8mVi05kP0Ea/n/UzcSHcTJQfNg== + dependencies: + follow-redirects "^1.14.0" + +axios@^1.0.0, axios@^1.1.3, axios@^1.4.0, axios@^1.5.0, axios@^1.6.2: version "1.6.3" resolved "https://registry.yarnpkg.com/axios/-/axios-1.6.3.tgz#7f50f23b3aa246eff43c54834272346c396613f4" integrity sha512-fWyNdeawGam70jXSVlKl+SUNVcL6j6W79CuSIPfi6HnDUmSCH6gyUys/HrqHeA/wU0Az41rRgean494d0Jb+ww== @@ -8306,9 +8253,9 @@ caseless@~0.12.0: integrity sha512-4tYFyifaFfGacoiObjJegolkwSU4xQNGbVgUiNYVUxbQ2x2lUsFvY4hVgVzGiIe6WLOPqycWXA40l+PWsxthUw== chai@^4.3.7: - version "4.4.1" - resolved "https://registry.yarnpkg.com/chai/-/chai-4.4.1.tgz#3603fa6eba35425b0f2ac91a009fe924106e50d1" - integrity sha512-13sOfMv2+DWduEU+/xbun3LScLoqN17nBeTLUsmDfKdoiC1fr0n9PU4guu4AhRcOVFk/sW8LyZWHuhWtQZiF+g== + version "4.5.0" + resolved "https://registry.yarnpkg.com/chai/-/chai-4.5.0.tgz#707e49923afdd9b13a8b0b47d33d732d13812fd8" + integrity sha512-RITGBfijLkBddZvnn8jdqoTypxvqbOLYQkGGxXzeFjVHvudaPw0HNFD9x928/eUwYWd2dPCugVqspGALTZZQKw== dependencies: assertion-error "^1.1.0" check-error "^1.0.3" @@ -8316,7 +8263,7 @@ chai@^4.3.7: get-func-name "^2.0.2" loupe "^2.3.6" pathval "^1.1.1" - type-detect "^4.0.8" + type-detect "^4.1.0" chai@^5.1.1: version "5.1.1" @@ -9625,9 +9572,9 @@ dedent@^1.0.0: integrity sha512-7glNLfvdsMzZm3FpRY1CHuI2lbYDR+71YmrhmTZjYFD5pfT0ACgnGRdrrC9Mk2uICnzkcdelCx5at787UDGOvg== deep-eql@^4.1.3: - version "4.1.3" - resolved "https://registry.yarnpkg.com/deep-eql/-/deep-eql-4.1.3.tgz#7c7775513092f7df98d8df9996dd085eb668cc6d" - integrity sha512-WaEtAOpRA1MQ0eohqZjpGD8zdI0Ovsm8mmFhaDN8dvDZzyoUMcYDnf5Y6iu7HTXxf8JDS23qWa4a+hKCDyOPzw== + version "4.1.4" + resolved "https://registry.yarnpkg.com/deep-eql/-/deep-eql-4.1.4.tgz#d0d3912865911bb8fac5afb4e3acfa6a28dc72b7" + integrity sha512-SUwdGfqdKOwxCPeVYjwSyRpJ7Z+fhpwIAtmCUdZIWZ/YP5R9WAsyuSgpLVDi9bjWoN2LXHNss/dk3urXtdQxGg== dependencies: type-detect "^4.0.0" @@ -9637,15 +9584,16 @@ deep-eql@^5.0.1: integrity sha512-h5k/5U50IJJFpzfL6nO9jaaumfjO/f2NjK/oYB2Djzm4p9L+3T9qWpZqZ2hAbLPuuYq9wrU08WQyBTL5GbPk5Q== deep-equal@^2.0.5: - version "2.2.0" - resolved "https://registry.yarnpkg.com/deep-equal/-/deep-equal-2.2.0.tgz#5caeace9c781028b9ff459f33b779346637c43e6" - integrity sha512-RdpzE0Hv4lhowpIUKKMJfeH6C1pXdtT1/it80ubgWqwI3qpuxUBpC1S4hnHg+zjnuOoDkzUtUCEEkG+XG5l3Mw== + version "2.2.3" + resolved "https://registry.yarnpkg.com/deep-equal/-/deep-equal-2.2.3.tgz#af89dafb23a396c7da3e862abc0be27cf51d56e1" + integrity sha512-ZIwpnevOurS8bpT4192sqAowWM76JDKSHYzMLty3BZGSswgq6pBaH3DhCSW5xVAZICZyKdOBPjwww5wfgT/6PA== dependencies: - call-bind "^1.0.2" - es-get-iterator "^1.1.2" - get-intrinsic "^1.1.3" + array-buffer-byte-length "^1.0.0" + call-bind "^1.0.5" + es-get-iterator "^1.1.3" + get-intrinsic "^1.2.2" is-arguments "^1.1.1" - is-array-buffer "^3.0.1" + is-array-buffer "^3.0.2" is-date-object "^1.0.5" is-regex "^1.1.4" is-shared-array-buffer "^1.0.2" @@ -9653,11 +9601,11 @@ deep-equal@^2.0.5: object-is "^1.1.5" object-keys "^1.1.1" object.assign "^4.1.4" - regexp.prototype.flags "^1.4.3" + regexp.prototype.flags "^1.5.1" side-channel "^1.0.4" which-boxed-primitive "^1.0.2" which-collection "^1.0.1" - which-typed-array "^1.1.9" + which-typed-array "^1.1.13" deep-equal@~1.0.1: version "1.0.1" @@ -10619,7 +10567,7 @@ es-errors@^1.2.1, es-errors@^1.3.0: resolved "https://registry.yarnpkg.com/es-errors/-/es-errors-1.3.0.tgz#05f75a25dab98e4fb1dcd5e1472c0546d5057c8f" integrity sha512-Zf5H2Kxt2xjTvbJvP2ZWLEICxA6j+hAmMzIlypy4xcBg1vKVnx89Wy0GbS+kf5cwCVFFzdCFh2XSCFNULS6csw== -es-get-iterator@^1.1.2: +es-get-iterator@^1.1.3: version "1.1.3" resolved "https://registry.yarnpkg.com/es-get-iterator/-/es-get-iterator-1.1.3.tgz#3ef87523c5d464d41084b2c3c9c214f1199763d6" integrity sha512-sPZmqHBe6JIiTfN5q2pEi//TwxmAFHwj/XEuYjTuse78i8KxaqMTTzxPoFKuzRpDpTJ+0NAbpfenkmH2rePtuw== @@ -11633,6 +11581,11 @@ fn.name@1.x.x: resolved "https://registry.yarnpkg.com/fn.name/-/fn.name-1.1.0.tgz#26cad8017967aea8731bc42961d04a3d5988accc" integrity sha512-GRnmB5gPyJpAhTQdSZTSp9uaPSvl09KoYcMQtsB9rQoOmzs9dH6ffeccH+Z+cv6P68Hu5bC6JjRh4Ah/mHSNRw== +follow-redirects@^1.14.0, follow-redirects@^1.14.4: + version "1.15.9" + resolved "https://registry.yarnpkg.com/follow-redirects/-/follow-redirects-1.15.9.tgz#a604fa10e443bf98ca94228d9eebcc2e8a2c8ee1" + integrity sha512-gew4GsXizNgdoRyqmyfMHyAmXsZDk6mHkSxZFCzW9gwlbtOW44CDtYavM+y+72qD/Vq2l550kMF52DT8fOLJqQ== + follow-redirects@^1.15.0: version "1.15.6" resolved "https://registry.yarnpkg.com/follow-redirects/-/follow-redirects-1.15.6.tgz#7f815c0cda4249c74ff09e95ef97c23b5fd0399b" @@ -11936,7 +11889,7 @@ get-func-name@^2.0.1, get-func-name@^2.0.2: resolved "https://registry.yarnpkg.com/get-func-name/-/get-func-name-2.0.2.tgz#0d7cf20cd13fda808669ffa88f4ffc7a3943fc41" integrity sha512-8vXOvuE167CtIc3OyItco7N/dpRtBbYOsPsXCz7X/PMnlGjYjSGuZJgM1Y7mmew7BKf9BqvLX2tnOVy1BBUsxQ== -get-intrinsic@^1.1.1, get-intrinsic@^1.1.3, get-intrinsic@^1.2.1, get-intrinsic@^1.2.3, get-intrinsic@^1.2.4: +get-intrinsic@^1.1.3, get-intrinsic@^1.2.1, get-intrinsic@^1.2.2, get-intrinsic@^1.2.3, get-intrinsic@^1.2.4: version "1.2.4" resolved "https://registry.yarnpkg.com/get-intrinsic/-/get-intrinsic-1.2.4.tgz#e385f5a4b5227d449c3eabbad05494ef0abbeadd" integrity sha512-5uYhsJH8VJBTv7oslg4BznJYhDoRI6waYCxMmCdnTrcCrHA/fCFKoTFz2JKKE0HdDFUF7/oQuhzumXJK7paBRQ== @@ -12726,7 +12679,12 @@ http-assert@^1.3.0: deep-equal "~1.0.1" http-errors "~1.8.0" -http-cache-semantics@3.8.1, http-cache-semantics@4.1.1, http-cache-semantics@^4.0.0, http-cache-semantics@^4.1.0, http-cache-semantics@^4.1.1: +http-cache-semantics@3.8.1: + version "3.8.1" + resolved "https://registry.yarnpkg.com/http-cache-semantics/-/http-cache-semantics-3.8.1.tgz#39b0e16add9b605bf0a9ef3d9daaf4843b4cacd2" + integrity sha512-5ai2iksyV8ZXmnZhHH4rWPoxxistEexSi5936zIQ1bnNTW5VnA85B6P/VpXiRM017IgRvb2kKo1a//y+0wSp3w== + +http-cache-semantics@^4.0.0, http-cache-semantics@^4.1.0, http-cache-semantics@^4.1.1: version "4.1.1" resolved "https://registry.yarnpkg.com/http-cache-semantics/-/http-cache-semantics-4.1.1.tgz#abe02fcb2985460bf0323be664436ec3476a6d5a" integrity sha512-er295DKPVsV82j5kw1Gjt+ADA/XYHsajl82cGNQG2eyoPkvgUhX+nDIyelzhIWbbsXP39EHcI6l5tYs2FYqYXQ== @@ -13211,7 +13169,7 @@ is-arguments@^1.1.1: call-bind "^1.0.2" has-tostringtag "^1.0.0" -is-array-buffer@^3.0.1, is-array-buffer@^3.0.4: +is-array-buffer@^3.0.2, is-array-buffer@^3.0.4: version "3.0.4" resolved "https://registry.yarnpkg.com/is-array-buffer/-/is-array-buffer-3.0.4.tgz#7a1f92b3d61edd2bc65d24f130530ea93d7fae98" integrity sha512-wcjaerHw0ydZwfhiKbXJWLDY8A7yV7KhjQOpb83hGgGfId/aQa4TOvwyzn2PuswW2gPCYEL/nEAiSVpdOj1lXw== @@ -13402,10 +13360,10 @@ is-lambda@^1.0.1: resolved "https://registry.yarnpkg.com/is-lambda/-/is-lambda-1.0.1.tgz#3d9877899e6a53efc0160504cde15f82e6f061d5" integrity sha512-z7CMFGNrENq5iFB9Bqo64Xk6Y9sg+epq1myIcdHaGnbMTYOxvzsEtdYqQUylB7LxfkvgrrjP32T6Ywciio9UIQ== -is-map@^2.0.1, is-map@^2.0.2: - version "2.0.2" - resolved "https://registry.yarnpkg.com/is-map/-/is-map-2.0.2.tgz#00922db8c9bf73e81b7a335827bc2a43f2b91127" - integrity sha512-cOZFQQozTha1f4MxLFzlgKYPTyj26picdZTx82hbc/Xf4K/tZOOXSCkMvU4pKioRXGDLJRn0GM7Upe7kR721yg== +is-map@^2.0.2, is-map@^2.0.3: + version "2.0.3" + resolved "https://registry.yarnpkg.com/is-map/-/is-map-2.0.3.tgz#ede96b7fe1e270b3c4465e3a465658764926d62e" + integrity sha512-1Qed0/Hr2m+YqxnM09CjA2d/i6YZNfF6R2oRAOj36eUdS6qIV/huPJNSEpKbupewFs+ZsJlxsjjPbc0/afW6Lw== is-module@^1.0.0: version "1.0.0" @@ -13552,10 +13510,10 @@ is-self-closing@^1.0.1: dependencies: self-closing-tags "^1.0.1" -is-set@^2.0.1, is-set@^2.0.2: - version "2.0.2" - resolved "https://registry.yarnpkg.com/is-set/-/is-set-2.0.2.tgz#90755fa4c2562dc1c5d4024760d6119b94ca18ec" - integrity sha512-+2cnTEZeY5z/iXGbLhPrOAaK/Mau5k5eXq9j14CpRTftq0pAJu2MwVRSZhyZWBzx3o6X795Lz6Bpb6R0GKf37g== +is-set@^2.0.2, is-set@^2.0.3: + version "2.0.3" + resolved "https://registry.yarnpkg.com/is-set/-/is-set-2.0.3.tgz#8ab209ea424608141372ded6e0cb200ef1d9d01d" + integrity sha512-iPAjerrse27/ygGLxw+EBR9agv9Y6uLeYVJMu+QNCoouJ1/1ri0mGrcWpfCqFZuzzx3WjtwxG098X+n4OuRkPg== is-shared-array-buffer@^1.0.2, is-shared-array-buffer@^1.0.3: version "1.0.3" @@ -13653,10 +13611,10 @@ is-utf8@^0.2.0: resolved "https://registry.yarnpkg.com/is-utf8/-/is-utf8-0.2.1.tgz#4b0da1442104d1b336340e80797e865cf39f7d72" integrity sha512-rMYPYvCzsXywIsldgLaSoPlw5PfoB/ssr7hY4pLfcodrA5M/eArza1a9VmTiNIBNMjOGr1Ow9mTyU2o69U6U9Q== -is-weakmap@^2.0.1: - version "2.0.1" - resolved "https://registry.yarnpkg.com/is-weakmap/-/is-weakmap-2.0.1.tgz#5008b59bdc43b698201d18f62b37b2ca243e8cf2" - integrity sha512-NSBR4kH5oVj1Uwvv970ruUkCV7O1mzgVFO4/rev2cLRda9Tm9HrL70ZPut4rOHgY0FNrUu9BCbXA2sdQ+x0chA== +is-weakmap@^2.0.2: + version "2.0.2" + resolved "https://registry.yarnpkg.com/is-weakmap/-/is-weakmap-2.0.2.tgz#bf72615d649dfe5f699079c54b83e47d1ae19cfd" + integrity sha512-K5pXYOm9wqY1RgjpL3YTkF39tni1XajUIkawTLUo9EZEVUFga5gSQJF8nNS7ZwJQ02y+1YCNYcMh+HIf1ZqE+w== is-weakref@^1.0.2: version "1.0.2" @@ -13665,13 +13623,13 @@ is-weakref@^1.0.2: dependencies: call-bind "^1.0.2" -is-weakset@^2.0.1: - version "2.0.2" - resolved "https://registry.yarnpkg.com/is-weakset/-/is-weakset-2.0.2.tgz#4569d67a747a1ce5a994dfd4ef6dcea76e7c0a1d" - integrity sha512-t2yVvttHkQktwnNNmBQ98AhENLdPUTDTE21uPqAQ0ARwQfGeQKRVS0NNurH7bTf7RrvcVn1OOge45CnBeHCSmg== +is-weakset@^2.0.3: + version "2.0.3" + resolved "https://registry.yarnpkg.com/is-weakset/-/is-weakset-2.0.3.tgz#e801519df8c0c43e12ff2834eead84ec9e624007" + integrity sha512-LvIm3/KWzS9oRFHugab7d+M/GcBXuXX5xZkzPmN+NxihdQlZUQ4dWuSV1xR/sq6upL1TJEDrfBgRepHFdBtSNQ== dependencies: - call-bind "^1.0.2" - get-intrinsic "^1.1.1" + call-bind "^1.0.7" + get-intrinsic "^1.2.4" is-whitespace@^0.3.0: version "0.3.0" @@ -13737,11 +13695,6 @@ isobject@^3.0.1: resolved "https://registry.yarnpkg.com/isobject/-/isobject-3.0.1.tgz#4e431e92b11a9731636aa1f9c8d1ccbcfdab78df" integrity sha512-WhB9zCku7EGTj/HQQRz5aUQEUeoQZH2bWcltRErOpymJ4boYE6wL9Tbr23krRPSZ+C5zqNSrSw+Cc7sZZ4b7vg== -isobject@^4.0.0: - version "4.0.0" - resolved "https://registry.yarnpkg.com/isobject/-/isobject-4.0.0.tgz#3f1c9155e73b192022a80819bacd0343711697b0" - integrity sha512-S/2fF5wH8SJA/kmwr6HYhK/RI/OkhD84k8ntalo0iJjZikgq1XFvR5M8NPT1x5F7fBwCG3qHfnzeP/Vh/ZxCUA== - isolated-vm@^4.7.2: version "4.7.2" resolved "https://registry.yarnpkg.com/isolated-vm/-/isolated-vm-4.7.2.tgz#5670d5cce1d92004f9b825bec5b0b11fc7501b65" @@ -14395,9 +14348,9 @@ jsdom@^16.0.1: xml-name-validator "^3.0.0" jsdom@^21.1.1: - version "21.1.1" - resolved "https://registry.yarnpkg.com/jsdom/-/jsdom-21.1.1.tgz#ab796361e3f6c01bcfaeda1fea3c06197ac9d8ae" - integrity sha512-Jjgdmw48RKcdAIQyUD1UdBh2ecH7VqwaXPN3ehoZN6MqgVbMn+lRm1aAT1AsdJRAJpwfa4IpwgzySn61h2qu3w== + version "21.1.2" + resolved "https://registry.yarnpkg.com/jsdom/-/jsdom-21.1.2.tgz#6433f751b8718248d646af1cdf6662dc8a1ca7f9" + integrity sha512-sCpFmK2jv+1sjff4u7fzft+pUh2KSUbUrEHYHyfSIbGTIcmnjyp83qg6qLwdJ/I3LpTXx33ACxeRL7Lsyc6lGQ== dependencies: abab "^2.0.6" acorn "^8.8.2" @@ -14412,7 +14365,7 @@ jsdom@^21.1.1: http-proxy-agent "^5.0.0" https-proxy-agent "^5.0.1" is-potential-custom-element-name "^1.0.1" - nwsapi "^2.2.2" + nwsapi "^2.2.4" parse5 "^7.1.2" rrweb-cssom "^0.6.0" saxes "^6.0.0" @@ -16191,7 +16144,7 @@ mkdirp@^1.0.3, mkdirp@^1.0.4: resolved "https://registry.yarnpkg.com/mkdirp/-/mkdirp-1.0.4.tgz#3eb5ed62622756d79a5f0e2a221dfebad75c2f7e" integrity sha512-vVqVZQyf3WLx2Shd0qJ9xuvqgAyKPLAiqITEtqW0oIUjzo3PePDd6fW9iFz30ef7Ysp/oiWqbhszeGWW2T6Gzw== -mlly@^1.1.0, mlly@^1.7.0: +mlly@^1.1.0, mlly@^1.7.1: version "1.7.1" resolved "https://registry.yarnpkg.com/mlly/-/mlly-1.7.1.tgz#e0336429bb0731b6a8e887b438cbdae522c8f32f" integrity sha512-rrVRZRELyQzrIUAVMHxP97kv+G786pHmOKzuFII8zDYahFBS7qnHh2AlYSl1GAHhaMPCz6/oHjVMcfFYgFYHgA== @@ -16301,7 +16254,7 @@ msgpackr-extract@^3.0.2: "@msgpackr-extract/msgpackr-extract-linux-x64" "3.0.2" "@msgpackr-extract/msgpackr-extract-win32-x64" "3.0.2" -msgpackr@1.10.1, msgpackr@^1.5.2: +msgpackr@^1.5.2: version "1.10.1" resolved "https://registry.yarnpkg.com/msgpackr/-/msgpackr-1.10.1.tgz#51953bb4ce4f3494f0c4af3f484f01cfbb306555" integrity sha512-r5VRLv9qouXuLiIBrLpl2d5ZvPt8svdQTl5/vMvE4nzDMyEX4sgW5yWhuBBj5UmgwOTWj8CIdSXn5sAfsHAWIQ== @@ -16495,13 +16448,25 @@ node-domexception@1.0.0: resolved "https://registry.yarnpkg.com/node-domexception/-/node-domexception-1.0.0.tgz#6888db46a1f71c0b76b3f7555016b63fe64766e5" integrity sha512-/jKZoMpw0F8GRwl4/eLROPA3cfcXtLApP0QzLmUT/HuPCZWyB7IY9ZrMeKw2O/nFIqPQB3PVM9aYm0F312AXDQ== -node-fetch@2.6.0, node-fetch@2.6.7, node-fetch@^2.6.0, node-fetch@^2.6.1, node-fetch@^2.6.7, node-fetch@^2.6.9, node-fetch@^2.7.0: +node-fetch@2.6.0: + version "2.6.0" + resolved "https://registry.yarnpkg.com/node-fetch/-/node-fetch-2.6.0.tgz#e633456386d4aa55863f676a7ab0daa8fdecb0fd" + integrity sha512-8dG4H5ujfvFiqDmVu9fQ5bOHUC15JMjMY/Zumv26oOvvVJjM67KF8koCWIabKQ1GJIa9r2mMZscBq/TbdOcmNA== + +node-fetch@2.6.7, node-fetch@^2.6.0, node-fetch@^2.6.1, node-fetch@^2.6.7: version "2.6.7" resolved "https://registry.yarnpkg.com/node-fetch/-/node-fetch-2.6.7.tgz#24de9fba827e3b4ae44dc8b20256a379160052ad" integrity sha512-ZjMPFEfVx5j+y2yF35Kzx5sF7kDzxuDj6ziH4FFbOp87zKDZNx8yExJIb05OGF4Nlt9IHFIMBkRl41VdvcNdbQ== dependencies: whatwg-url "^5.0.0" +node-fetch@^2.6.9, node-fetch@^2.7.0: + version "2.7.0" + resolved "https://registry.yarnpkg.com/node-fetch/-/node-fetch-2.7.0.tgz#d0f0fa6e3e2dc1d27efcd8ad99d550bda94d187d" + integrity sha512-c4FRfUm/dbcWZ7U+1Wq0AwCyFL+3nt2bEw05wfxSz+DWpWsitgmSgYmy2dQdWyKC1694ELPqMs/YzUSNozLt8A== + dependencies: + whatwg-url "^5.0.0" + node-forge@^1.2.1, node-forge@^1.3.1: version "1.3.1" resolved "https://registry.yarnpkg.com/node-forge/-/node-forge-1.3.1.tgz#be8da2af243b2417d5f646a770663a92b7e9ded3" @@ -16849,11 +16814,16 @@ nunjucks@^3.2.3: asap "^2.0.3" commander "^5.1.0" -nwsapi@^2.2.0, nwsapi@^2.2.2: +nwsapi@^2.2.0: version "2.2.4" resolved "https://registry.yarnpkg.com/nwsapi/-/nwsapi-2.2.4.tgz#fd59d5e904e8e1f03c25a7d5a15cfa16c714a1e5" integrity sha512-NHj4rzRo0tQdijE9ZqAx6kYDcoRwYwSYzCA8MY3JzfxlrvEU0jhnhJT9BhqhJs7I/dKcrDm6TyulaRqZPIhN5g== +nwsapi@^2.2.4: + version "2.2.12" + resolved "https://registry.yarnpkg.com/nwsapi/-/nwsapi-2.2.12.tgz#fb6af5c0ec35b27b4581eb3bbad34ec9e5c696f8" + integrity sha512-qXDmcVlZV4XRtKFzddidpfVP4oMSGhga+xdMc25mv8kaLUHtgzCDhUxkrN8exkGdTlLNaXj7CV3GtON7zuGZ+w== + nx-cloud@16.0.5: version "16.0.5" resolved "https://registry.yarnpkg.com/nx-cloud/-/nx-cloud-16.0.5.tgz#fa0b0185d254405ec47fcbcdbbd8b12ff1add096" @@ -16954,12 +16924,12 @@ object-inspect@^1.13.1: integrity sha512-5qoj1RUiKOMsCCNLV1CBiPYE10sziTsnmNxkAI/rZhiD63CF7IqdFGC/XzjWjpSgLf0LxXX3bDFIh0E18f6UhQ== object-is@^1.1.5: - version "1.1.5" - resolved "https://registry.yarnpkg.com/object-is/-/object-is-1.1.5.tgz#b9deeaa5fc7f1846a0faecdceec138e5778f53ac" - integrity sha512-3cyDsyHgtmi7I7DfSSI2LDp6SK2lwvtbg0p0R1e0RvTqF5ceGx+K2dfSjm1bKDMVCFEDAQvy+o8c6a7VujOddw== + version "1.1.6" + resolved "https://registry.yarnpkg.com/object-is/-/object-is-1.1.6.tgz#1a6a53aed2dd8f7e6775ff870bea58545956ab07" + integrity sha512-F8cZ+KfGlSGi09lJT7/Nd6KJZ9ygtvYC0/UYYLI9nmQKLMnydpB9yvbv9K1uSkEu7FU9vYPmVwLg328tX+ot3Q== dependencies: - call-bind "^1.0.2" - define-properties "^1.1.3" + call-bind "^1.0.7" + define-properties "^1.2.1" object-keys@^1.1.1: version "1.1.1" @@ -17647,7 +17617,15 @@ passport-strategy@1.x.x, passport-strategy@^1.0.0: resolved "https://registry.yarnpkg.com/passport-strategy/-/passport-strategy-1.0.0.tgz#b5539aa8fc225a3d1ad179476ddf236b440f52e4" integrity sha512-CB97UUvDKJde2V0KDWWB3lyf6PC3FaZP7YxZ2G8OAtn9p4HI9j9JLP9qjOGZFvyl8uwNT8qM+hGnz/n16NI7oA== -passport@0.6.0, passport@^0.4.0, passport@^0.6.0: +passport@^0.4.0: + version "0.4.1" + resolved "https://registry.yarnpkg.com/passport/-/passport-0.4.1.tgz#941446a21cb92fc688d97a0861c38ce9f738f270" + integrity sha512-IxXgZZs8d7uFSt3eqNjM9NQ3g3uQCW5avD8mRNoXV99Yig50vjuaez6dQK2qC0kVWPRTujxY0dWgGfT09adjYg== + dependencies: + passport-strategy "1.x.x" + pause "0.0.1" + +passport@^0.6.0: version "0.6.0" resolved "https://registry.yarnpkg.com/passport/-/passport-0.6.0.tgz#e869579fab465b5c0b291e841e6cc95c005fac9d" integrity sha512-0fe+p3ZnrWRW74fe8+SvCyf4a3Pb2/h7gFkQ8yTJpAO50gDzlfjZUZTO1k5Eg9kUct22OxHLqDZoKUWRHOh9ug== @@ -18024,12 +18002,12 @@ pkg-dir@^4.2.0: find-up "^4.0.0" pkg-types@^1.1.1: - version "1.1.1" - resolved "https://registry.yarnpkg.com/pkg-types/-/pkg-types-1.1.1.tgz#07b626880749beb607b0c817af63aac1845a73f2" - integrity sha512-ko14TjmDuQJ14zsotODv7dBlwxKhUKQEhuhmbqo1uCi9BB0Z2alo/wAXg6q1dTR5TyuqYyWhjtfe/Tsh+X28jQ== + version "1.2.0" + resolved "https://registry.yarnpkg.com/pkg-types/-/pkg-types-1.2.0.tgz#d0268e894e93acff11a6279de147e83354ebd42d" + integrity sha512-+ifYuSSqOQ8CqP4MbZA5hDpb97n3E8SVWdJe+Wms9kj745lmd3b7EZJiqvmLwAlmRfjrI7Hi5z3kdBJ93lFNPA== dependencies: confbox "^0.1.7" - mlly "^1.7.0" + mlly "^1.7.1" pathe "^1.1.2" pkginfo@0.4.x: @@ -18920,7 +18898,7 @@ pseudomap@^1.0.2: resolved "https://registry.yarnpkg.com/pseudomap/-/pseudomap-1.0.2.tgz#f052a28da70e618917ef0a8ac34c1ae5a68286b3" integrity sha512-b/YwNhb8lk1Zz2+bXXpS/LK9OisiZZ1SNsSLxN1x2OXVEhW2Ckr/7mWE5vrC1ZTiJlD9g19jWszTmJsB+oEpFQ== -psl@^1.1.33: +psl@^1.1.28, psl@^1.1.33: version "1.9.0" resolved "https://registry.yarnpkg.com/psl/-/psl-1.9.0.tgz#d0df2a137f00794565fcaf3b2c00cd09f8d5a5a7" integrity sha512-E/ZsdU4HLs/68gYzgGTkMicWTLPdAftJLfJFlLUAAKZGkStNU72sZjT66SnMDVOfOWY/YAoiD7Jxa9iHvngcag== @@ -19396,7 +19374,7 @@ regexp-tree@~0.1.1: resolved "https://registry.yarnpkg.com/regexp-tree/-/regexp-tree-0.1.27.tgz#2198f0ef54518ffa743fe74d983b56ffd631b6cd" integrity sha512-iETxpjK6YoRWJG5o6hXLwvjYAoW+FEZn9os0PD/b6AP6xQwsa/Y7lCVgIixBbUPMfhu+i2LtdeAqVTgGlQarfA== -regexp.prototype.flags@^1.4.3, regexp.prototype.flags@^1.5.2: +regexp.prototype.flags@^1.5.1, regexp.prototype.flags@^1.5.2: version "1.5.2" resolved "https://registry.yarnpkg.com/regexp.prototype.flags/-/regexp.prototype.flags-1.5.2.tgz#138f644a3350f981a858c44f6bb1a61ff59be334" integrity sha512-NcDiDkTLuPR+++OCKB0nWafEmhg/Da8aUPLPMQbK+bxKKCm1/S5he+AqYa4PlMCVBalb4/yxIRub6qkEx5yJbw== @@ -19520,6 +19498,11 @@ requires-port@^1.0.0: resolved "https://registry.yarnpkg.com/requires-port/-/requires-port-1.0.0.tgz#925d2601d39ac485e091cf0da5c6e694dc3dcaff" integrity sha512-KigOCHcocU3XODJxsu8i/j8T9tzT4adHiecwORRQ0ZZFcp7ahwXuRU1m+yuO90C5ZUyGeGfocHDI14M3L3yDAQ== +resize-observer-polyfill@^1.5.1: + version "1.5.1" + resolved "https://registry.yarnpkg.com/resize-observer-polyfill/-/resize-observer-polyfill-1.5.1.tgz#0e9020dd3d21024458d4ebd27e23e40269810464" + integrity sha512-LwZrotdHOo12nQuZlHEmtuXdqGoOD0OhaxopaNFxWzInpEgaLWoVuAMbTzixuosCx2nEG58ngzW3vxdWoxIgdg== + resolve-alpn@^1.0.0: version "1.2.1" resolved "https://registry.yarnpkg.com/resolve-alpn/-/resolve-alpn-1.2.1.tgz#b7adbdac3546aaaec20b45e7d8265927072726f9" @@ -19987,6 +19970,11 @@ sax@1.2.1: resolved "https://registry.yarnpkg.com/sax/-/sax-1.2.1.tgz#7b8e656190b228e81a66aea748480d828cd2d37a" integrity sha512-8I2a3LovHTOpm7NV5yOyO8IHqgVsfK4+UuySrXU8YXkSRX7k6hCV9b3HrkKCr3nMpgj+0bmocaJJWpvp1oc7ZA== +sax@>=0.1.1: + version "1.4.1" + resolved "https://registry.yarnpkg.com/sax/-/sax-1.4.1.tgz#44cc8988377f126304d3b3fc1010c733b929ef0f" + integrity sha512-+aWOz7yVScEGoKNd4PA10LZ8sk0A/z5+nXQG5giUO5rprX9jgYsTdov9qCchZiPIZezbZH+jRut8nPodFAX4Jg== + sax@>=0.6.0: version "1.2.4" resolved "https://registry.yarnpkg.com/sax/-/sax-1.2.4.tgz#2816234e2378bddc4e5354fab5caa895df7100d9" @@ -20059,13 +20047,33 @@ semver-diff@^3.1.1: dependencies: semver "^6.3.0" -"semver@2 || 3 || 4 || 5", semver@7.5.3, semver@^5.5.0, semver@^5.6.0, semver@^5.7.1, semver@^6.0.0, semver@^6.1.1, semver@^6.1.2, semver@^6.2.0, semver@^6.3.0, semver@^6.3.1, semver@^7.0.0, semver@^7.1.1, semver@^7.3.2, semver@^7.3.4, semver@^7.3.5, semver@^7.3.7, semver@^7.3.8, semver@^7.5.3, semver@^7.5.4, semver@^7.6.0, semver@^7.6.3, semver@~2.3.1: +"semver@2 || 3 || 4 || 5", semver@^5.5.0, semver@^5.6.0, semver@^5.7.1: + version "5.7.2" + resolved "https://registry.yarnpkg.com/semver/-/semver-5.7.2.tgz#48d55db737c3287cd4835e17fa13feace1c41ef8" + integrity sha512-cBznnQ9KjJqU67B52RMC65CMarK2600WFnbkcaiwWq3xy/5haFJlshgnpjovMVJ+Hff49d8GEn0b87C5pDQ10g== + +semver@7.5.3, semver@^7.0.0, semver@^7.1.1, semver@^7.3.2, semver@^7.3.4, semver@^7.3.5, semver@^7.3.7, semver@^7.3.8, semver@^7.5.3: version "7.5.3" resolved "https://registry.yarnpkg.com/semver/-/semver-7.5.3.tgz#161ce8c2c6b4b3bdca6caadc9fa3317a4c4fe88e" integrity sha512-QBlUtyVk/5EeHbi7X0fw6liDZc7BBmEaSYn01fMU1OUYbf6GPsbTtd8WmnqbI20SeycoHSeiybkE/q1Q+qlThQ== dependencies: lru-cache "^6.0.0" +semver@^6.0.0, semver@^6.1.1, semver@^6.1.2, semver@^6.2.0, semver@^6.3.0, semver@^6.3.1: + version "6.3.1" + resolved "https://registry.yarnpkg.com/semver/-/semver-6.3.1.tgz#556d2ef8689146e46dcea4bfdd095f3434dffcb4" + integrity sha512-BR7VvDCVHO+q2xBEWskxS6DJE1qRnb7DxzUrogb71CWoSficBxYsiAGd+Kl0mmq/MprG9yArRkyrQxTO6XjMzA== + +semver@^7.5.4, semver@^7.6.0, semver@^7.6.3: + version "7.6.3" + resolved "https://registry.yarnpkg.com/semver/-/semver-7.6.3.tgz#980f7b5550bc175fb4dc09403085627f9eb33143" + integrity sha512-oVekP1cKtI+CTDvHWYFUcMtsK/00wmAEfyqKfNdARm8u1wNVhSgaX7A8d4UuIlUI5e84iEwOhs7ZPYRmzU9U6A== + +semver@~2.3.1: + version "2.3.2" + resolved "https://registry.yarnpkg.com/semver/-/semver-2.3.2.tgz#b9848f25d6cf36333073ec9ef8856d42f1233e52" + integrity sha512-abLdIKCosKfpnmhS52NCTjO4RiLspDfsn37prjzGrp9im5DPJOgh82Os92vtwGh6XdQryKI/7SREZnV+aqiXrA== + seq-queue@^0.0.5: version "0.0.5" resolved "https://registry.yarnpkg.com/seq-queue/-/seq-queue-0.0.5.tgz#d56812e1c017a6e4e7c3e3a37a1da6d78dd3c93e" @@ -20743,16 +20751,7 @@ string-similarity@^4.0.4: resolved "https://registry.yarnpkg.com/string-similarity/-/string-similarity-4.0.4.tgz#42d01ab0b34660ea8a018da8f56a3309bb8b2a5b" integrity sha512-/q/8Q4Bl4ZKAPjj8WerIBJWALKkaPRfrvhfF8k/B23i4nzrlRj2/go1m90In7nG/3XDSbOo0+pu6RvCTM9RGMQ== -"string-width-cjs@npm:string-width@^4.2.0": - version "4.2.3" - resolved "https://registry.yarnpkg.com/string-width/-/string-width-4.2.3.tgz#269c7117d27b05ad2e536830a8ec895ef9c6d010" - integrity sha512-wKyQRQpjJ0sIp62ErSZdGsjMJWsap5oRNihHhu6G7JVO/9jIB6UyevL+tXuOqrng8j/cxKTWyWUwvSTriiZz/g== - dependencies: - emoji-regex "^8.0.0" - is-fullwidth-code-point "^3.0.0" - strip-ansi "^6.0.1" - -"string-width@^1.0.2 || 2 || 3 || 4", string-width@^4.0.0, string-width@^4.1.0, string-width@^4.2.0, string-width@^4.2.2, string-width@^4.2.3: +"string-width-cjs@npm:string-width@^4.2.0", "string-width@^1.0.2 || 2 || 3 || 4", string-width@^4.0.0, string-width@^4.1.0, string-width@^4.2.0, string-width@^4.2.2, string-width@^4.2.3: version "4.2.3" resolved "https://registry.yarnpkg.com/string-width/-/string-width-4.2.3.tgz#269c7117d27b05ad2e536830a8ec895ef9c6d010" integrity sha512-wKyQRQpjJ0sIp62ErSZdGsjMJWsap5oRNihHhu6G7JVO/9jIB6UyevL+tXuOqrng8j/cxKTWyWUwvSTriiZz/g== @@ -20843,7 +20842,7 @@ stringify-object@^3.2.1: is-obj "^1.0.1" is-regexp "^1.0.0" -"strip-ansi-cjs@npm:strip-ansi@^6.0.1": +"strip-ansi-cjs@npm:strip-ansi@^6.0.1", strip-ansi@^6.0.0, strip-ansi@^6.0.1: version "6.0.1" resolved "https://registry.yarnpkg.com/strip-ansi/-/strip-ansi-6.0.1.tgz#9e26c63d30f53443e9489495b2105d37b67a85d9" integrity sha512-Y38VPSHcqkFrCpFnQ9vuSXmquuv5oXOKpGeT6aGrr3o3Gc9AlVa6JBfUSOCnbxGGZF+/0ooI7KrPuUSztUdU5A== @@ -20857,13 +20856,6 @@ strip-ansi@^5.0.0, strip-ansi@^5.1.0, strip-ansi@^5.2.0: dependencies: ansi-regex "^4.1.0" -strip-ansi@^6.0.0, strip-ansi@^6.0.1: - version "6.0.1" - resolved "https://registry.yarnpkg.com/strip-ansi/-/strip-ansi-6.0.1.tgz#9e26c63d30f53443e9489495b2105d37b67a85d9" - integrity sha512-Y38VPSHcqkFrCpFnQ9vuSXmquuv5oXOKpGeT6aGrr3o3Gc9AlVa6JBfUSOCnbxGGZF+/0ooI7KrPuUSztUdU5A== - dependencies: - ansi-regex "^5.0.1" - strip-ansi@^7.0.1: version "7.0.1" resolved "https://registry.yarnpkg.com/strip-ansi/-/strip-ansi-7.0.1.tgz#61740a08ce36b61e50e65653f07060d000975fb2" @@ -21523,12 +21515,7 @@ tiny-queue@^0.2.0: resolved "https://registry.yarnpkg.com/tiny-queue/-/tiny-queue-0.2.1.tgz#25a67f2c6e253b2ca941977b5ef7442ef97a6046" integrity sha512-EijGsv7kzd9I9g0ByCl6h42BWNGUZrlCSejfrb3AKeHC33SGbASu1VDf5O3rRiiUOhAC9CHdZxFPbZu0HmR70A== -tinybench@^2.3.1: - version "2.8.0" - resolved "https://registry.yarnpkg.com/tinybench/-/tinybench-2.8.0.tgz#30e19ae3a27508ee18273ffed9ac7018949acd7b" - integrity sha512-1/eK7zUnIklz4JUUlL+658n58XO2hHLQfSk1Zf2LKieUjxidN16eKFEoDEfjHc3ohofSSqK3X5yO6VGb6iW8Lw== - -tinybench@^2.8.0: +tinybench@^2.3.1, tinybench@^2.8.0: version "2.9.0" resolved "https://registry.yarnpkg.com/tinybench/-/tinybench-2.9.0.tgz#103c9f8ba6d7237a47ab6dd1dcff77251863426b" integrity sha512-0+DUvqWMValLmha6lr4kD8iAMK1HzV0/aKnCtWb9v9641TnP/MFb7Pc2bxoxQjTXAErryXVgUOfv2YqNllqGeg== @@ -21649,7 +21636,7 @@ touch@^3.1.0: dependencies: nopt "~1.0.10" -tough-cookie@4.1.3, "tough-cookie@^2.3.3 || ^3.0.1 || ^4.0.0", tough-cookie@^4.0.0, tough-cookie@^4.1.2, tough-cookie@~2.5.0: +"tough-cookie@^2.3.3 || ^3.0.1 || ^4.0.0", tough-cookie@^4.0.0, tough-cookie@^4.1.2: version "4.1.3" resolved "https://registry.yarnpkg.com/tough-cookie/-/tough-cookie-4.1.3.tgz#97b9adb0728b42280aa3d814b6b999b2ff0318bf" integrity sha512-aX/y5pVRkfRnfmuX+OdbSdXvPe6ieKX/G2s7e98f4poJHnqH3281gDPm/metm6E/WRamfx7WC4HUqkWHfQHprw== @@ -21659,6 +21646,14 @@ tough-cookie@4.1.3, "tough-cookie@^2.3.3 || ^3.0.1 || ^4.0.0", tough-cookie@^4.0 universalify "^0.2.0" url-parse "^1.5.3" +tough-cookie@~2.5.0: + version "2.5.0" + resolved "https://registry.yarnpkg.com/tough-cookie/-/tough-cookie-2.5.0.tgz#cd9fb2a0aa1d5a12b473bd9fb96fa3dcff65ade2" + integrity sha512-nlLsUzgm1kfLXSXfRZMc1KLAugd4hqJHDTvc2hDIwS3mZAfMEuMbc03SujMF+GEcpaX/qboeycw6iO8JwVv2+g== + dependencies: + psl "^1.1.28" + punycode "^2.1.1" + tr46@^2.1.0: version "2.1.0" resolved "https://registry.yarnpkg.com/tr46/-/tr46-2.1.0.tgz#fa87aa81ca5d5941da8cbf1f9b749dc969a4e240" @@ -21873,11 +21868,16 @@ type-check@~0.3.2: dependencies: prelude-ls "~1.1.2" -type-detect@4.0.8, type-detect@^4.0.0, type-detect@^4.0.8: +type-detect@4.0.8: version "4.0.8" resolved "https://registry.yarnpkg.com/type-detect/-/type-detect-4.0.8.tgz#7646fb5f18871cfbb7749e69bd39a6388eb7450c" integrity sha512-0fr/mIH1dlO+x7TlcMy+bIDqKPsw/70tVyeHW787goQjhmqaZe10uwLujubK9q9Lg6Fiho1KUKDYz0Z7k7g5/g== +type-detect@^4.0.0, type-detect@^4.1.0: + version "4.1.0" + resolved "https://registry.yarnpkg.com/type-detect/-/type-detect-4.1.0.tgz#deb2453e8f08dcae7ae98c626b13dddb0155906c" + integrity sha512-Acylog8/luQ8L7il+geoSxhEkazvkslg7PSNKOX59mbB9cOveP5aq9h74Y7YU8yDpJwetzQQrfIwtf4Wp4LKcw== + type-fest@^0.13.1: version "0.13.1" resolved "https://registry.yarnpkg.com/type-fest/-/type-fest-0.13.1.tgz#0172cb5bce80b0bd542ea348db50c7e21834d934" @@ -22042,9 +22042,9 @@ typo-js@*: integrity sha512-C7pYBQK17EjSg8tVNY91KHdUt5Nf6FMJ+c3js076quPmBML57PmNMzAcIq/2kf/hSYtFABNDIYNYlJRl5BJhGw== ufo@^1.5.3: - version "1.5.3" - resolved "https://registry.yarnpkg.com/ufo/-/ufo-1.5.3.tgz#3325bd3c977b6c6cd3160bf4ff52989adc9d3344" - integrity sha512-Y7HYmWaFwPUmkoQCUIAYpKqkOf+SbVj/2fJJZ4RJMCfZp0rTGwRbzQD+HghfnhKOjL9E01okqz+ncJskGYfBNw== + version "1.5.4" + resolved "https://registry.yarnpkg.com/ufo/-/ufo-1.5.4.tgz#16d6949674ca0c9e0fbbae1fa20a71d7b1ded754" + integrity sha512-UsUk3byDzKd04EyoZ7U4DOlxQaD14JUKQl6/P7wiX4FNvUfm3XL246n9W5AmqwW5RSFJ27NAuM0iLscAOYUiGQ== uglify-js@^3.1.4: version "3.17.4" @@ -22182,14 +22182,6 @@ unpipe@1.0.0: resolved "https://registry.yarnpkg.com/unpipe/-/unpipe-1.0.0.tgz#b2bf4ee8514aae6165b4817829d21b2ef49904ec" integrity sha512-pjy2bYhSsufwWlKwPc+l3cN7+wuJlK6uz0YdJEOlQDbl6jo/YlPi4mb8agUkVC8BF7V8NuzeyPNqRksA3hztKQ== -unset-value@2.0.1: - version "2.0.1" - resolved "https://registry.yarnpkg.com/unset-value/-/unset-value-2.0.1.tgz#57bed0c22d26f28d69acde5df9a11b77c74d2df3" - integrity sha512-2hvrBfjUE00PkqN+q0XP6yRAOGrR06uSiUoIQGZkc7GxvQ9H7v8quUPNtZjMg4uux69i8HWpIjLPUKwCuRGyNg== - dependencies: - has-value "^2.0.2" - isobject "^4.0.0" - untildify@^4.0.0: version "4.0.0" resolved "https://registry.yarnpkg.com/untildify/-/untildify-4.0.0.tgz#2bc947b953652487e4600949fb091e3ae8cd919b" @@ -22703,21 +22695,21 @@ which-boxed-primitive@^1.0.2: is-symbol "^1.0.3" which-collection@^1.0.1: - version "1.0.1" - resolved "https://registry.yarnpkg.com/which-collection/-/which-collection-1.0.1.tgz#70eab71ebbbd2aefaf32f917082fc62cdcb70906" - integrity sha512-W8xeTUwaln8i3K/cY1nGXzdnVZlidBcagyNFtBdD5kxnb4TvGKR7FfSIS3mYpwWS1QUCutfKz8IY8RjftB0+1A== + version "1.0.2" + resolved "https://registry.yarnpkg.com/which-collection/-/which-collection-1.0.2.tgz#627ef76243920a107e7ce8e96191debe4b16c2a0" + integrity sha512-K4jVyjnBdgvc86Y6BkaLZEN933SwYOuBFkdmBu9ZfkcAbdVbpITnDmjvZ/aQjRXQrv5EPkTnD1s39GiiqbngCw== dependencies: - is-map "^2.0.1" - is-set "^2.0.1" - is-weakmap "^2.0.1" - is-weakset "^2.0.1" + is-map "^2.0.3" + is-set "^2.0.3" + is-weakmap "^2.0.2" + is-weakset "^2.0.3" which-module@^2.0.0: version "2.0.0" resolved "https://registry.yarnpkg.com/which-module/-/which-module-2.0.0.tgz#d9ef07dce77b9902b8a3a8fa4b31c3e3f7e6e87a" integrity sha512-B+enWhmw6cjfVC7kS8Pj9pCrKSc5txArRyaYGe088shv/FGWH+0Rjx/xPgtsWfsUtS27FkP697E4DDhgrgoc0Q== -which-typed-array@^1.1.14, which-typed-array@^1.1.15, which-typed-array@^1.1.9: +which-typed-array@^1.1.13, which-typed-array@^1.1.14, which-typed-array@^1.1.15: version "1.1.15" resolved "https://registry.yarnpkg.com/which-typed-array/-/which-typed-array-1.1.15.tgz#264859e9b11a649b388bfaaf4f767df1f779b38d" integrity sha512-oV0jmFtUky6CXfkqehVvBP/LSWJ2sy4vWMioiENyJLePrBO/yKyV9OyJySfAKosh+RYkIl5zJCNZ8/4JncrpdA== @@ -22749,15 +22741,7 @@ which@^3.0.0: dependencies: isexe "^2.0.0" -why-is-node-running@^2.2.2: - version "2.2.2" - resolved "https://registry.yarnpkg.com/why-is-node-running/-/why-is-node-running-2.2.2.tgz#4185b2b4699117819e7154594271e7e344c9973e" - integrity sha512-6tSwToZxTOcotxHeA+qGCq1mVzKR3CwcJGmVcY+QE8SHy6TnpFnh8PAvPNHYr7EcuVeG0QSMxtYCuO1ta/G/oA== - dependencies: - siginfo "^2.0.0" - stackback "0.0.2" - -why-is-node-running@^2.3.0: +why-is-node-running@^2.2.2, why-is-node-running@^2.3.0: version "2.3.0" resolved "https://registry.yarnpkg.com/why-is-node-running/-/why-is-node-running-2.3.0.tgz#a3f69a97107f494b3cdc3bdddd883a7d65cebf04" integrity sha512-hUrmaWBdVDcxvYqnyh09zunKzROWjbZTiNy8dBEjkS7ehEDQibXJ7XvlmtbwuTclUiIyN+CyXQD4Vmko8fNm8w== @@ -22832,7 +22816,7 @@ worker-farm@1.7.0: dependencies: errno "~0.1.7" -"wrap-ansi-cjs@npm:wrap-ansi@^7.0.0": +"wrap-ansi-cjs@npm:wrap-ansi@^7.0.0", wrap-ansi@^7.0.0: version "7.0.0" resolved "https://registry.yarnpkg.com/wrap-ansi/-/wrap-ansi-7.0.0.tgz#67e145cff510a6a6984bdf1152911d69d2eb9e43" integrity sha512-YVGIj2kamLSTxw6NsZjoBxfSwsn0ycdesmc4p+Q21c5zPuZ1pl+NfxVdxPtdHvmNVOQ6XSYG4AUtyt/Fi7D16Q== @@ -22850,15 +22834,6 @@ wrap-ansi@^5.1.0: string-width "^3.0.0" strip-ansi "^5.0.0" -wrap-ansi@^7.0.0: - version "7.0.0" - resolved "https://registry.yarnpkg.com/wrap-ansi/-/wrap-ansi-7.0.0.tgz#67e145cff510a6a6984bdf1152911d69d2eb9e43" - integrity sha512-YVGIj2kamLSTxw6NsZjoBxfSwsn0ycdesmc4p+Q21c5zPuZ1pl+NfxVdxPtdHvmNVOQ6XSYG4AUtyt/Fi7D16Q== - dependencies: - ansi-styles "^4.0.0" - string-width "^4.1.0" - strip-ansi "^6.0.0" - wrap-ansi@^8.1.0: version "8.1.0" resolved "https://registry.yarnpkg.com/wrap-ansi/-/wrap-ansi-8.1.0.tgz#56dc22368ee570face1b49819975d9b9a5ead214" @@ -22941,7 +22916,12 @@ ws@^7.4.6: resolved "https://registry.yarnpkg.com/ws/-/ws-7.5.10.tgz#58b5c20dc281633f6c19113f39b349bd8bd558d9" integrity sha512-+dbF1tHwZpXcbOJdVOkzLDxZP1ailvSxM6ZweXTegylPny803bFhA+vqBYw4s31NSAk4S2Qz+AKXK9a4wkdjcQ== -ws@^8.13.0, ws@~8.17.1: +ws@^8.13.0: + version "8.18.0" + resolved "https://registry.yarnpkg.com/ws/-/ws-8.18.0.tgz#0d7505a6eafe2b0e712d232b42279f53bc289bbc" + integrity sha512-8VbfWfHLbbwu3+N6OKsOMpBdT4kXPDDB9cJk2bJ6mh9ucxdlnNvH1e+roYkKmN9Nxw2yjz7VzeO9oOz2zJ04Pw== + +ws@~8.17.1: version "8.17.1" resolved "https://registry.yarnpkg.com/ws/-/ws-8.17.1.tgz#9293da530bb548febc95371d90f9c878727d919b" integrity sha512-6XQFvXTkbfUOZOKKILFG1PDK2NDQs4azKQl26T0YS5CxqWLgXajbPZ+h4gZekJyRqFU8pvnbAbbs/3TgRPy+GQ== @@ -22981,10 +22961,33 @@ xml-parse-from-string@^1.0.0: resolved "https://registry.yarnpkg.com/xml-parse-from-string/-/xml-parse-from-string-1.0.1.tgz#a9029e929d3dbcded169f3c6e28238d95a5d5a28" integrity sha512-ErcKwJTF54uRzzNMXq2X5sMIy88zJvfN2DmdoQvy7PAFJ+tPRU6ydWuOKNMyfmOjdyBQTFREi60s0Y0SyI0G0g== -xml2js@0.1.x, xml2js@0.4.19, xml2js@0.5.0, xml2js@0.6.2, xml2js@^0.4.19, xml2js@^0.4.5: - version "0.6.2" - resolved "https://registry.yarnpkg.com/xml2js/-/xml2js-0.6.2.tgz#dd0b630083aa09c161e25a4d0901e2b2a929b499" - integrity sha512-T4rieHaC1EXcES0Kxxj4JWgaUQHDk+qwHcYOCFHfiwKz7tOVPLq7Hjq9dM1WCMhylqMEfP7hMcOIChvotiZegA== +xml2js@0.1.x: + version "0.1.14" + resolved "https://registry.yarnpkg.com/xml2js/-/xml2js-0.1.14.tgz#5274e67f5a64c5f92974cd85139e0332adc6b90c" + integrity sha512-pbdws4PPPNc1HPluSUKamY4GWMk592K7qwcj6BExbVOhhubub8+pMda/ql68b6L3luZs/OGjGSB5goV7SnmgnA== + dependencies: + sax ">=0.1.1" + +xml2js@0.4.19: + version "0.4.19" + resolved "https://registry.yarnpkg.com/xml2js/-/xml2js-0.4.19.tgz#686c20f213209e94abf0d1bcf1efaa291c7827a7" + integrity sha512-esZnJZJOiJR9wWKMyuvSE1y6Dq5LCuJanqhxslH2bxM6duahNZ+HMpCLhBQGZkbX6xRf8x1Y2eJlgt2q3qo49Q== + dependencies: + sax ">=0.6.0" + xmlbuilder "~9.0.1" + +xml2js@0.5.0: + version "0.5.0" + resolved "https://registry.yarnpkg.com/xml2js/-/xml2js-0.5.0.tgz#d9440631fbb2ed800203fad106f2724f62c493b7" + integrity sha512-drPFnkQJik/O+uPKpqSgr22mpuFHqKdbS835iAQrUC73L2F5WkboIRd63ai/2Yg6I1jzifPFKH2NTK+cfglkIA== + dependencies: + sax ">=0.6.0" + xmlbuilder "~11.0.0" + +xml2js@^0.4.19, xml2js@^0.4.5: + version "0.4.23" + resolved "https://registry.yarnpkg.com/xml2js/-/xml2js-0.4.23.tgz#a0c69516752421eb2ac758ee4d4ccf58843eac66" + integrity sha512-ySPiMjM0+pLDftHgXY4By0uswI3SPKLDw/i3UXbnO8M/p28zqexCUoPmQFrYD+/1BzhGJSs2i1ERWKJAtiLrug== dependencies: sax ">=0.6.0" xmlbuilder "~11.0.0" @@ -22994,6 +22997,11 @@ xmlbuilder@~11.0.0: resolved "https://registry.yarnpkg.com/xmlbuilder/-/xmlbuilder-11.0.1.tgz#be9bae1c8a046e76b31127726347d0ad7002beb3" integrity sha512-fDlsI/kFEx7gLvbecc0/ohLG50fugQp8ryHzMTuW9vSa1GJ0XYWKnhsUx7oie3G98+r56aTQIUB4kht42R3JvA== +xmlbuilder@~9.0.1: + version "9.0.7" + resolved "https://registry.yarnpkg.com/xmlbuilder/-/xmlbuilder-9.0.7.tgz#132ee63d2ec5565c557e20f4c22df9aca686b10d" + integrity sha512-7YXTQc3P2l9+0rjaUbLwMKRhtmwg1M1eDf6nag7urC7pIPYLD9W/jmzQ4ptRSUbodw5S0jfoGTflLemQibSpeQ== + xmlchars@^2.2.0: version "2.2.0" resolved "https://registry.yarnpkg.com/xmlchars/-/xmlchars-2.2.0.tgz#060fe1bcb7f9c76fe2a17db86a9bc3ab894210cb" @@ -23172,9 +23180,9 @@ yocto-queue@^0.1.0: integrity sha512-rVksvsnNCdJ/ohGc6xgPwyN8eheCxsiLM8mxuE/t/mOVqJewPuO1miLpTHQiRgTKCLexL4MeAFVagts7HmNZ2Q== yocto-queue@^1.0.0: - version "1.0.0" - resolved "https://registry.yarnpkg.com/yocto-queue/-/yocto-queue-1.0.0.tgz#7f816433fb2cbc511ec8bf7d263c3b58a1a3c251" - integrity sha512-9bnSc/HEW2uRy67wc+T8UwauLuPJVn28jb+GtJY16iiKWyvmYJRXVT4UamsAEGQfPohgr2q4Tq0sQbQlxTfi1g== + version "1.1.1" + resolved "https://registry.yarnpkg.com/yocto-queue/-/yocto-queue-1.1.1.tgz#fef65ce3ac9f8a32ceac5a634f74e17e5b232110" + integrity sha512-b4JR1PFR10y1mKjhHY9LaGo6tmrgjit7hxVIeAmyMw3jegXR4dhYqLaQF5zMXZxY7tLpMyJeLjr1C4rLmkVe8g== yup@0.32.9: version "0.32.9" From 3c58a593f95f93cb047e724d71c5cc95fa0aee78 Mon Sep 17 00:00:00 2001 From: Sam Rose Date: Fri, 6 Sep 2024 15:03:17 +0100 Subject: [PATCH 40/85] Improve typing around in-memory search. --- .../server/src/integrations/googlesheets.ts | 2 +- .../integrations/tests/googlesheets.spec.ts | 184 ++++-------------- .../integrations/tests/utils/googlesheets.ts | 93 +++++++++ packages/shared-core/src/filters.ts | 53 +++-- 4 files changed, 158 insertions(+), 174 deletions(-) create mode 100644 packages/server/src/integrations/tests/utils/googlesheets.ts diff --git a/packages/server/src/integrations/googlesheets.ts b/packages/server/src/integrations/googlesheets.ts index b257a5da9b..d8d775c629 100644 --- a/packages/server/src/integrations/googlesheets.ts +++ b/packages/server/src/integrations/googlesheets.ts @@ -580,7 +580,7 @@ export class GoogleSheetsIntegration implements DatasourcePlus { let response = [] for (let row of filtered) { response.push( - this.buildRowObject(headerValues, row.toObject(), row._rowNumber) + this.buildRowObject(headerValues, row.toObject(), row["_rowNumber"]) ) } diff --git a/packages/server/src/integrations/tests/googlesheets.spec.ts b/packages/server/src/integrations/tests/googlesheets.spec.ts index 685345660d..80899699bc 100644 --- a/packages/server/src/integrations/tests/googlesheets.spec.ts +++ b/packages/server/src/integrations/tests/googlesheets.spec.ts @@ -1,36 +1,22 @@ import { setEnv as setCoreEnv } from "@budibase/backend-core" -import type { GoogleSpreadsheetWorksheet } from "google-spreadsheet" import nock from "nock" -import { structures } from "@budibase/backend-core/tests" import TestConfiguration from "../../tests/utilities/TestConfiguration" -import { GoogleSheetsConfig, GoogleSheetsIntegration } from "../googlesheets" import { Datasource, FieldType, SourceName, - Table, - TableSchema, TableSourceType, } from "@budibase/types" -import { generateDatasourceID } from "../../db/utils" +import { access } from "node:fs" +import { GoogleSheetsMock } from "./utils/googlesheets" describe("Google Sheets Integration", () => { const config = new TestConfiguration() - let integration: GoogleSheetsIntegration let cleanupEnv: () => void - let table: Table let datasource: Datasource - - const datasourceConfig: GoogleSheetsConfig = { - spreadsheetId: "randomId", - auth: { - appId: "appId", - accessToken: "accessToken", - refreshToken: "refreshToken", - }, - } + let mock: GoogleSheetsMock beforeAll(async () => { cleanupEnv = setCoreEnv({ @@ -38,11 +24,20 @@ describe("Google Sheets Integration", () => { GOOGLE_CLIENT_SECRET: "test", }) + await config.init() + datasource = await config.api.datasource.create({ name: "Test Datasource", type: "datasource", source: SourceName.GOOGLE_SHEETS, - config: datasourceConfig, + config: { + spreadsheetId: "randomId", + auth: { + appId: "appId", + accessToken: "accessToken", + refreshToken: "refreshToken", + }, + }, }) }) @@ -52,139 +47,40 @@ describe("Google Sheets Integration", () => { }) beforeEach(async () => { - await config.init() - - integration = new GoogleSheetsIntegration(datasourceConfig) - - table = await config.api.table.save({ - name: "Test Table", - type: "table", - sourceId: generateDatasourceID(), - sourceType: TableSourceType.EXTERNAL, - schema: { - name: { - name: "name", - type: FieldType.STRING, - constraints: { - type: "string", - }, - }, - description: { - name: "description", - type: FieldType.STRING, - constraints: { - type: "string", - }, - }, - }, - }) - nock.cleanAll() - nock("https://www.googleapis.com/").post("/oauth2/v4/token").reply(200, { - grant_type: "client_credentials", - client_id: "your-client-id", - client_secret: "your-client-secret", - }) + mock = GoogleSheetsMock.forDatasource(datasource) + mock.mockAuth() }) - function createBasicTable(name: string, columns: string[]): Table { - return { - type: "table", - name, - sourceId: generateDatasourceID(), - sourceType: TableSourceType.EXTERNAL, - schema: { - ...columns.reduce((p, c) => { - p[c] = { - name: c, + describe("create", () => { + it("creates a new table", async () => { + nock("https://sheets.googleapis.com/", { + reqheaders: { authorization: "Bearer test" }, + }) + .get("/v4/spreadsheets/randomId/") + .reply(200, {}) + + const table = await config.api.table.save({ + name: "Test Table", + type: "table", + sourceId: datasource._id!, + sourceType: TableSourceType.EXTERNAL, + schema: { + name: { + name: "name", type: FieldType.STRING, constraints: { type: "string", }, - } - return p - }, {} as TableSchema), - }, - } - } - - function createSheet({ - headerValues, - }: { - headerValues: string[] - }): GoogleSpreadsheetWorksheet { - return { - // to ignore the unmapped fields - ...({} as any), - loadHeaderRow: jest.fn(), - headerValues, - setHeaderRow: jest.fn(), - } - } - - describe("update table", () => { - it("adding a new field will be adding a new header row", async () => { - await config.doInContext(structures.uuid(), async () => { - const tableColumns = ["name", "description", "new field"] - const table = createBasicTable(structures.uuid(), tableColumns) - - const sheet = createSheet({ headerValues: ["name", "description"] }) - sheetsByTitle[table.name] = sheet - await integration.updateTable(table) - - expect(sheet.loadHeaderRow).toHaveBeenCalledTimes(1) - expect(sheet.setHeaderRow).toHaveBeenCalledTimes(1) - expect(sheet.setHeaderRow).toHaveBeenCalledWith(tableColumns) - }) - }) - - it("removing an existing field will remove the header from the google sheet", async () => { - const sheet = await config.doInContext(structures.uuid(), async () => { - const tableColumns = ["name"] - const table = createBasicTable(structures.uuid(), tableColumns) - - const sheet = createSheet({ - headerValues: ["name", "description", "location"], - }) - sheetsByTitle[table.name] = sheet - await integration.updateTable(table) - return sheet - }) - expect(sheet.loadHeaderRow).toHaveBeenCalledTimes(1) - expect(sheet.setHeaderRow).toHaveBeenCalledTimes(1) - expect(sheet.setHeaderRow).toHaveBeenCalledWith([ - "name", - "description", - "location", - ]) - }) - }) - - describe("getTableNames", () => { - it("can fetch table names", async () => { - await config.doInContext(structures.uuid(), async () => { - const sheetNames: string[] = [] - for (let i = 0; i < 5; i++) { - const sheet = createSheet({ headerValues: [] }) - sheetsByIndex.push(sheet) - sheetNames.push(sheet.title) - } - - const res = await integration.getTableNames() - - expect(mockGoogleIntegration.loadInfo).toHaveBeenCalledTimes(1) - expect(res).toEqual(sheetNames) - }) - }) - }) - - describe("testConnection", () => { - it("can test successful connections", async () => { - await config.doInContext(structures.uuid(), async () => { - const res = await integration.testConnection() - - expect(mockGoogleIntegration.loadInfo).toHaveBeenCalledTimes(1) - expect(res).toEqual({ connected: true }) + }, + description: { + name: "description", + type: FieldType.STRING, + constraints: { + type: "string", + }, + }, + }, }) }) }) diff --git a/packages/server/src/integrations/tests/utils/googlesheets.ts b/packages/server/src/integrations/tests/utils/googlesheets.ts new file mode 100644 index 0000000000..f19a50ee76 --- /dev/null +++ b/packages/server/src/integrations/tests/utils/googlesheets.ts @@ -0,0 +1,93 @@ +import { Datasource } from "@budibase/types" +import nock from "nock" +import { GoogleSheetsConfig } from "../../googlesheets" + +interface ErrorValue { + type: string + message: string +} + +interface ExtendedValue { + stringValue?: string + numberValue?: number + boolValue?: boolean + formulaValue?: string + errorValue?: ErrorValue +} + +interface CellData { + userEnteredValue: ExtendedValue +} + +interface RowData { + values: CellData[] +} + +interface GridData { + startRow: number + startColumn: number + rowData: RowData[] +} + +interface Sheet { + properties: { + sheetId: string + title: string + } + data: GridData[] +} + +interface Spreadsheet { + spreadsheetId: string + sheets: Sheet[] +} + +export class GoogleSheetsMock { + private config: GoogleSheetsConfig + private sheet: Spreadsheet + + static forDatasource(datasource: Datasource): GoogleSheetsMock { + return new GoogleSheetsMock(datasource.config as GoogleSheetsConfig) + } + + private constructor(config: GoogleSheetsConfig) { + this.config = config + this.sheet = { + spreadsheetId: config.spreadsheetId, + sheets: [], + } + } + + init() { + nock("https://www.googleapis.com/").post("/oauth2/v4/token").reply(200, { + grant_type: "client_credentials", + client_id: "your-client-id", + client_secret: "your-client-secret", + }) + nock("https://oauth2.googleapis.com/") + .post("/token", { + client_id: "test", + client_secret: "test", + grant_type: "refresh_token", + refresh_token: "refreshToken", + }) + .reply(200, { + access_token: "test", + expires_in: 3600, + token_type: "Bearer", + scopes: "https://www.googleapis.com/auth/spreadsheets", + }) + + nock("https://sheets.googleapis.com/", { + reqheaders: { authorization: "Bearer test" }, + }) + .get("/v4/spreadsheets/randomId/") + .reply(200, {}) + + nock("https://sheets.googleapis.com/", { + reqheaders: { authorization: "Bearer test" }, + }) + .post(`/v4/spreadsheets/${this.config.spreadsheetId}/:batchUpdate`) + .reply(200, () => {}) + } +} diff --git a/packages/shared-core/src/filters.ts b/packages/shared-core/src/filters.ts index 70fb24b373..bfe8dc1aa1 100644 --- a/packages/shared-core/src/filters.ts +++ b/packages/shared-core/src/filters.ts @@ -448,10 +448,10 @@ export function fixupFilterArrays(filters: SearchFilters) { return filters } -export const search = ( - docs: Record[], +export function search( + docs: Record[], query: RowSearchParams -): SearchResponse> => { +): SearchResponse> { let result = runQuery(docs, query.query) if (query.sort) { result = sort(result, query.sort, query.sortOrder || SortOrder.ASCENDING) @@ -475,11 +475,11 @@ export const search = ( * from custom doc type e.g. Google Sheets * */ -export const runQuery = ( - docs: Record[], +export function runQuery>( + docs: T[], query: SearchFilters, - findInDoc: Function = deepGet -) => { + findInDoc: (obj: T, key: string) => any = deepGet +): T[] { if (!docs || !Array.isArray(docs)) { return [] } @@ -502,7 +502,7 @@ export const runQuery = ( type: SearchFilterOperator, test: (docValue: any, testValue: any) => boolean ) => - (doc: Record) => { + (doc: T) => { for (const [key, testValue] of Object.entries(query[type] || {})) { const valueToCheck = isLogicalSearchOperator(type) ? doc @@ -749,11 +749,8 @@ export const runQuery = ( } ) - const docMatch = (doc: Record) => { - const filterFunctions: Record< - SearchFilterOperator, - (doc: Record) => boolean - > = { + const docMatch = (doc: T) => { + const filterFunctions: Record boolean> = { string: stringMatch, fuzzy: fuzzyMatch, range: rangeMatch, @@ -797,12 +794,12 @@ export const runQuery = ( * @param sortOrder the sort order ("ascending" or "descending") * @param sortType the type of sort ("string" or "number") */ -export const sort = ( - docs: any[], - sort: string, +export function sort>( + docs: T[], + sort: keyof T, sortOrder: SortOrder, sortType = SortType.STRING -) => { +): T[] { if (!sort || !sortOrder || !sortType) { return docs } @@ -817,19 +814,17 @@ export const sort = ( return parseFloat(x) } - return docs - .slice() - .sort((a: { [x: string]: any }, b: { [x: string]: any }) => { - const colA = parse(a[sort]) - const colB = parse(b[sort]) + return docs.slice().sort((a, b) => { + const colA = parse(a[sort]) + const colB = parse(b[sort]) - const result = colB == null || colA > colB ? 1 : -1 - if (sortOrder.toLowerCase() === "descending") { - return result * -1 - } + const result = colB == null || colA > colB ? 1 : -1 + if (sortOrder.toLowerCase() === "descending") { + return result * -1 + } - return result - }) + return result + }) } /** @@ -838,7 +833,7 @@ export const sort = ( * @param docs the data * @param limit the number of docs to limit to */ -export const limit = (docs: any[], limit: string) => { +export function limit(docs: T[], limit: string): T[] { const numLimit = parseFloat(limit) if (isNaN(numLimit)) { return docs From 891f73681539239f6930374cfb4fba885683e7ab Mon Sep 17 00:00:00 2001 From: Martin McKeaveney Date: Fri, 6 Sep 2024 15:37:09 +0100 Subject: [PATCH 41/85] unit tests for secrets mapping --- packages/builder/vite.config.mjs | 2 +- .../functions/backfill/global/configs.ts | 7 +- .../src/api/controllers/global/configs.ts | 2 +- .../controllers/global/tests/configs.spec.ts | 76 +++++++++++++++++++ 4 files changed, 81 insertions(+), 6 deletions(-) create mode 100644 packages/worker/src/api/controllers/global/tests/configs.spec.ts diff --git a/packages/builder/vite.config.mjs b/packages/builder/vite.config.mjs index a84fba2dae..f5ff388952 100644 --- a/packages/builder/vite.config.mjs +++ b/packages/builder/vite.config.mjs @@ -79,7 +79,7 @@ export default defineConfig(({ mode }) => { "process.env.POSTHOG_TOKEN": JSON.stringify(process.env.POSTHOG_TOKEN), }), copyFonts("fonts"), - ...(isProduction ? [] : devOnlyPlugins), + ...(isProduction ? [] : devOnlyPlugins), ], optimizeDeps: { exclude: ["@roxi/routify", "fsevents"], diff --git a/packages/server/src/migrations/functions/backfill/global/configs.ts b/packages/server/src/migrations/functions/backfill/global/configs.ts index bb0781eba3..04eb9caff2 100644 --- a/packages/server/src/migrations/functions/backfill/global/configs.ts +++ b/packages/server/src/migrations/functions/backfill/global/configs.ts @@ -15,12 +15,11 @@ import { } from "@budibase/types" import env from "./../../../../environment" -export function getConfigParams(type?: ConfigType): DatabaseQueryOpts { - const configType = type || "" +export function getConfigParams(): DatabaseQueryOpts { return { include_docs: true, - startkey: `${DocumentType.CONFIG}${SEPARATOR}${configType}`, - endkey: `${DocumentType.CONFIG}${SEPARATOR}${configType}${UNICODE_MAX}`, + startkey: `${DocumentType.CONFIG}${SEPARATOR}`, + endkey: `${DocumentType.CONFIG}${SEPARATOR}${UNICODE_MAX}`, } } diff --git a/packages/worker/src/api/controllers/global/configs.ts b/packages/worker/src/api/controllers/global/configs.ts index 087feede3c..4a19168c4f 100644 --- a/packages/worker/src/api/controllers/global/configs.ts +++ b/packages/worker/src/api/controllers/global/configs.ts @@ -199,7 +199,7 @@ async function verifyOIDCConfig(config: OIDCConfigs) { await verifySSOConfig(ConfigType.OIDC, config.configs[0]) } -async function verifyAIConfig(config: AIConfig, existingConfig?: AIConfig) { +export async function verifyAIConfig(config: AIConfig, existingConfig?: AIConfig) { if (!existingConfig) return // ensure that the redacted API keys are not overwritten in the DB diff --git a/packages/worker/src/api/controllers/global/tests/configs.spec.ts b/packages/worker/src/api/controllers/global/tests/configs.spec.ts new file mode 100644 index 0000000000..1a1d78757d --- /dev/null +++ b/packages/worker/src/api/controllers/global/tests/configs.spec.ts @@ -0,0 +1,76 @@ +import { TestConfiguration, structures } from "../../../../tests" +import { constants, configs } from "@budibase/backend-core" +import { AIConfig, UserCtx } from "@budibase/types" +import { find, verifyAIConfig } from "../configs" + +jest.mock("@budibase/backend-core", () => ({ + ...jest.requireActual("@budibase/backend-core"), + configs: { + getConfig: jest.fn(), + save: jest.fn() + }, +})) + +describe("Global configs controller", () => { + it("Should strip secrets when pulling AI config", async () => { + configs.getConfig.mockResolvedValue({ + config: { + ai: { + apiKey: "abc123APIKey", + baseUrl: "https://api.example.com", + } + } + }) + const ctx = { + params: { + type: "ai" + }, + throw: jest.fn() + } as UserCtx + + await find(ctx) + + expect(ctx.body).toEqual({ + config: { + ai: { + apiKey: "--secret-value--", + "baseUrl": "https://api.example.com" + } + } + }) + }) + + it("Should not update existing secrets when updating an existing AI Config", async () => { + const newConfig = { + type: "ai", + config: { + aiconfig: { + provider: "OpenAI", + isDefault: true, + name: "MyConfig", + active: true, + defaultModel: "gpt4", + apiKey: "--secret-value--" + } + } + } + + const existingConfig = { + type: "ai", + config: { + aiconfig: { + provider: "OpenAI", + isDefault: true, + name: "MyConfig", + active: true, + defaultModel: "gpt4", + apiKey: "myapikey" + } + } + } + + await verifyAIConfig(newConfig, existingConfig) + // should be unchanged + expect(newConfig.config.aiconfig.apiKey === "myapikey") + }) +}) From 6bf4f47110fae08a78b1d2413aa11f13a0936d52 Mon Sep 17 00:00:00 2001 From: Martin McKeaveney Date: Fri, 6 Sep 2024 15:42:37 +0100 Subject: [PATCH 42/85] further tests around default BB AI config --- .../controllers/global/tests/configs.spec.ts | 47 +++++++++++++++++-- 1 file changed, 44 insertions(+), 3 deletions(-) diff --git a/packages/worker/src/api/controllers/global/tests/configs.spec.ts b/packages/worker/src/api/controllers/global/tests/configs.spec.ts index 1a1d78757d..393a11a043 100644 --- a/packages/worker/src/api/controllers/global/tests/configs.spec.ts +++ b/packages/worker/src/api/controllers/global/tests/configs.spec.ts @@ -1,6 +1,6 @@ -import { TestConfiguration, structures } from "../../../../tests" -import { constants, configs } from "@budibase/backend-core" -import { AIConfig, UserCtx } from "@budibase/types" +import { configs } from "@budibase/backend-core" +import { UserCtx } from "@budibase/types" +import * as pro from "@budibase/pro" import { find, verifyAIConfig } from "../configs" jest.mock("@budibase/backend-core", () => ({ @@ -11,7 +11,12 @@ jest.mock("@budibase/backend-core", () => ({ }, })) + describe("Global configs controller", () => { + afterEach(() => { + jest.resetAllMocks() + }) + it("Should strip secrets when pulling AI config", async () => { configs.getConfig.mockResolvedValue({ config: { @@ -40,6 +45,42 @@ describe("Global configs controller", () => { }) }) + it("Should return the default BB AI config when the feature is turned on", async () => { + pro.features.isBudibaseAIEnabled = jest.fn(() => true) + configs.getConfig.mockResolvedValue({ + config: { + ai: { + apiKey: "abc123APIKey", + baseUrl: "https://api.example.com", + } + } + }) + const ctx = { + params: { + type: "ai" + }, + throw: jest.fn() + } as UserCtx + + await find(ctx) + + expect(ctx.body).toEqual({ + config: { + budibase_ai: { + provider: "OpenAI", + active: true, + isDefault: true, + defaultModel: undefined, + name: "Budibase AI", + }, + ai: { + apiKey: "--secret-value--", + "baseUrl": "https://api.example.com" + } + } + }) + }) + it("Should not update existing secrets when updating an existing AI Config", async () => { const newConfig = { type: "ai", From 073017a0d6bdbb9b84ae762c23905b2ff6bb55c1 Mon Sep 17 00:00:00 2001 From: Martin McKeaveney Date: Fri, 6 Sep 2024 15:43:34 +0100 Subject: [PATCH 43/85] remove todo comment --- packages/worker/src/api/routes/global/configs.ts | 1 - 1 file changed, 1 deletion(-) diff --git a/packages/worker/src/api/routes/global/configs.ts b/packages/worker/src/api/routes/global/configs.ts index 409e0a7797..471005c2aa 100644 --- a/packages/worker/src/api/routes/global/configs.ts +++ b/packages/worker/src/api/routes/global/configs.ts @@ -76,7 +76,6 @@ function aiValidation() { active: Joi.boolean().required(), baseUrl: Joi.string().optional().allow("", null), apiKey: Joi.string().required(), - // TODO: should be enum defaultModel: Joi.string().optional(), }).required() ) From b9499406534216549593aa555d6b7bddd1b68689 Mon Sep 17 00:00:00 2001 From: Martin McKeaveney Date: Fri, 6 Sep 2024 15:44:23 +0100 Subject: [PATCH 44/85] remove comments --- packages/backend-core/src/configs/configs.ts | 39 -------------------- 1 file changed, 39 deletions(-) diff --git a/packages/backend-core/src/configs/configs.ts b/packages/backend-core/src/configs/configs.ts index 17f4abfe24..1eafd9596e 100644 --- a/packages/backend-core/src/configs/configs.ts +++ b/packages/backend-core/src/configs/configs.ts @@ -256,42 +256,3 @@ export async function getSCIMConfig(): Promise { const config = await getConfig(ConfigType.SCIM) return config?.config } - - -// AI - -// TODO: Can we assume that you are licensed when you hit this endpoint? -// export async function getAIConfig(): Promise { - // if (!env.SELF_HOSTED) { - // // always use the env vars in cloud - // // TODO: Licensing stuff - make this right - // if (env.OPENAI_API_KEY) { - // return getDefaultBudibaseAIConfig() - // } - // } - - // prefer the config in self-host - // let config = await getConfig(ConfigType.AI) - - // fallback to env vars - // if (!config || !config.activated) { - // config = getDefaultBudibaseAIConfig() - // } - - // return config -// } - -// export function getDefaultBudibaseAIConfig(): AIInnerConfig | undefined { -// if (env.OPENAI_API_KEY) { -// return { -// provider: "", -// isDefault: true, -// name: "Budibase AI", -// active: true, -// baseUrl: "", -// apiKey: env.OPENAI_API_KEY, -// // TODO: should be enum -// defaultModel: "" -// } -// } -// } From 0359b20347afbd2aa1d5894db60e2987222cf533 Mon Sep 17 00:00:00 2001 From: Sam Rose Date: Fri, 6 Sep 2024 16:55:16 +0100 Subject: [PATCH 45/85] First new test passing. --- .../src/api/controllers/table/external.ts | 1 + .../integrations/tests/googlesheets.spec.ts | 14 +- .../integrations/tests/utils/googlesheets.ts | 333 +++++++++++++++++- 3 files changed, 335 insertions(+), 13 deletions(-) diff --git a/packages/server/src/api/controllers/table/external.ts b/packages/server/src/api/controllers/table/external.ts index c3356919c8..fbdb5c0b3a 100644 --- a/packages/server/src/api/controllers/table/external.ts +++ b/packages/server/src/api/controllers/table/external.ts @@ -53,6 +53,7 @@ export async function save( builderSocket?.emitDatasourceUpdate(ctx, datasource) return table } catch (err: any) { + throw err if (err instanceof Error) { ctx.throw(400, err.message) } else { diff --git a/packages/server/src/integrations/tests/googlesheets.spec.ts b/packages/server/src/integrations/tests/googlesheets.spec.ts index 80899699bc..21b7128999 100644 --- a/packages/server/src/integrations/tests/googlesheets.spec.ts +++ b/packages/server/src/integrations/tests/googlesheets.spec.ts @@ -49,17 +49,11 @@ describe("Google Sheets Integration", () => { beforeEach(async () => { nock.cleanAll() mock = GoogleSheetsMock.forDatasource(datasource) - mock.mockAuth() + mock.init() }) describe("create", () => { it("creates a new table", async () => { - nock("https://sheets.googleapis.com/", { - reqheaders: { authorization: "Bearer test" }, - }) - .get("/v4/spreadsheets/randomId/") - .reply(200, {}) - const table = await config.api.table.save({ name: "Test Table", type: "table", @@ -82,6 +76,12 @@ describe("Google Sheets Integration", () => { }, }, }) + + const cell = mock.getCell(table.name, "A1") + if (!cell) { + throw new Error("Cell not found") + } + expect(cell.userEnteredValue.stringValue).toEqual(table.name) }) }) }) diff --git a/packages/server/src/integrations/tests/utils/googlesheets.ts b/packages/server/src/integrations/tests/utils/googlesheets.ts index f19a50ee76..72c7665679 100644 --- a/packages/server/src/integrations/tests/utils/googlesheets.ts +++ b/packages/server/src/integrations/tests/utils/googlesheets.ts @@ -2,6 +2,82 @@ import { Datasource } from "@budibase/types" import nock from "nock" import { GoogleSheetsConfig } from "../../googlesheets" +type Value = string | number | boolean +type Dimension = "ROWS" | "COLUMNS" + +interface Range { + row: number | "ALL" + column: number | "ALL" +} + +interface DimensionProperties { + hiddenByFilter: boolean + hiddenByUser: boolean + pixelSize: number + // developerMetadata: DeveloperMetadata[] + // dataSourceColumnReference: DataSourceColumnReference +} + +interface ValueRange { + range: string + majorDimension: Dimension + values: Value[][] +} + +interface UpdateValuesResponse { + spreadsheetId: string + updatedRange: string + updatedRows: number + updatedColumns: number + updatedCells: number + updatedData: ValueRange +} + +interface AddSheetResponse { + properties: SheetProperties +} + +interface Response { + addSheet?: AddSheetResponse +} + +interface BatchUpdateResponse { + spreadsheetId: string + replies: Response[] + updatedSpreadsheet: Spreadsheet +} + +interface GridProperties { + rowCount: number + columnCount: number + frozenRowCount: number + frozenColumnCount: number + hideGridlines: boolean + rowGroupControlAfter: boolean + columnGroupControlAfter: boolean +} + +interface SheetProperties { + sheetId: number + title: string + gridProperties: GridProperties +} + +interface AddSheetRequest { + properties: SheetProperties +} + +interface Request { + addSheet?: AddSheetRequest +} + +interface BatchUpdateRequest { + requests: Request[] + includeSpreadsheetInResponse: boolean + responseRanges: string[] + responseIncludeGridData: boolean +} + interface ErrorValue { type: string message: string @@ -27,17 +103,21 @@ interface GridData { startRow: number startColumn: number rowData: RowData[] + rowMetadata: DimensionProperties[] + columnMetadata: DimensionProperties[] } interface Sheet { - properties: { - sheetId: string - title: string - } + properties: SheetProperties data: GridData[] } +interface SpreadsheetProperties { + title: string +} + interface Spreadsheet { + properties: SpreadsheetProperties spreadsheetId: string sheets: Sheet[] } @@ -53,6 +133,9 @@ export class GoogleSheetsMock { private constructor(config: GoogleSheetsConfig) { this.config = config this.sheet = { + properties: { + title: "Test Spreadsheet", + }, spreadsheetId: config.spreadsheetId, sheets: [], } @@ -82,12 +165,250 @@ export class GoogleSheetsMock { reqheaders: { authorization: "Bearer test" }, }) .get("/v4/spreadsheets/randomId/") - .reply(200, {}) + .reply(200, () => this.sheet) + .persist() nock("https://sheets.googleapis.com/", { reqheaders: { authorization: "Bearer test" }, }) .post(`/v4/spreadsheets/${this.config.spreadsheetId}/:batchUpdate`) - .reply(200, () => {}) + .reply(200, (uri: string, request: nock.Body): nock.Body => { + const batchUpdateRequest = request as BatchUpdateRequest + const replies: Response[] = [] + + for (const request of batchUpdateRequest.requests) { + if (request.addSheet) { + const properties: SheetProperties = { + title: request.addSheet.properties.title, + sheetId: this.sheet.sheets.length, + gridProperties: { + rowCount: 100, + columnCount: 26, + frozenRowCount: 0, + frozenColumnCount: 0, + hideGridlines: false, + rowGroupControlAfter: false, + columnGroupControlAfter: false, + }, + } + + this.sheet.sheets.push({ + properties, + data: [this.createEmptyGrid(100, 26)], + }) + + replies.push({ addSheet: { properties } }) + } + } + + const response: BatchUpdateResponse = { + spreadsheetId: this.sheet.spreadsheetId, + replies, + updatedSpreadsheet: this.sheet, + } + return response + }) + .persist() + + nock("https://sheets.googleapis.com/", { + reqheaders: { authorization: "Bearer test" }, + }) + .put( + new RegExp(`/v4/spreadsheets/${this.config.spreadsheetId}/values/.*`) + ) + .reply(200, (uri, request) => + this.handleValueUpdate(request as ValueRange) + ) + } + + private handleValueUpdate(valueRange: ValueRange): UpdateValuesResponse { + if (valueRange.majorDimension !== "ROWS") { + throw new Error("Only row-major updates are supported") + } + + const { sheet, topLeft, bottomRight } = this.parseA1Notation( + valueRange.range + ) + + if (topLeft.row === "ALL") { + topLeft.row = 0 + } + if (bottomRight.row === "ALL") { + bottomRight.row = sheet.properties.gridProperties.rowCount - 1 + } + if (topLeft.column === "ALL") { + topLeft.column = 0 + } + if (bottomRight.column === "ALL") { + bottomRight.column = sheet.properties.gridProperties.columnCount - 1 + } + + for (let row = topLeft.row; row <= bottomRight.row; row++) { + for ( + let column = topLeft.column; + column <= bottomRight.column; + column++ + ) { + const cell = this.getCellNumericIndexes(sheet, row, column) + if (!cell) { + continue + } + const value = + valueRange.values[row - topLeft.row][column - topLeft.column] + cell.userEnteredValue = this.createValue(value) + } + } + + const response: UpdateValuesResponse = { + spreadsheetId: this.sheet.spreadsheetId, + updatedRange: valueRange.range, + updatedRows: valueRange.values.length, + updatedColumns: valueRange.values[0].length, + updatedCells: valueRange.values.length * valueRange.values[0].length, + updatedData: valueRange, + } + return response + } + + private createValue(from: Value): ExtendedValue { + if (typeof from === "string") { + return { + stringValue: from, + } + } else if (typeof from === "number") { + return { + numberValue: from, + } + } else if (typeof from === "boolean") { + return { + boolValue: from, + } + } else { + throw new Error("Unsupported value type") + } + } + + private createEmptyGrid(numRows: number, numCols: number): GridData { + const rowData: RowData[] = [] + for (let row = 0; row < numRows; row++) { + const cells: CellData[] = [] + for (let col = 0; col < numCols; col++) { + cells.push({ + userEnteredValue: { + stringValue: "", + }, + }) + } + rowData.push({ + values: cells, + }) + } + const rowMetadata: DimensionProperties[] = [] + for (let row = 0; row < numRows; row++) { + rowMetadata.push({ + hiddenByFilter: false, + hiddenByUser: false, + pixelSize: 100, + }) + } + const columnMetadata: DimensionProperties[] = [] + for (let col = 0; col < numCols; col++) { + columnMetadata.push({ + hiddenByFilter: false, + hiddenByUser: false, + pixelSize: 100, + }) + } + + return { + startRow: 0, + startColumn: 0, + rowData, + rowMetadata, + columnMetadata, + } + } + + getCell(sheetName: string, ref: string): CellData | undefined { + const sheet = this.getSheetByName(sheetName) + if (!sheet) { + return undefined + } + const { row, column } = this.parseCell(ref) + if (row === "ALL" || column === "ALL") { + throw new Error("Only single cell references are supported") + } + return this.getCellNumericIndexes(sheet, row, column) + } + + private getCellNumericIndexes( + sheet: Sheet, + row: number, + column: number + ): CellData | undefined { + const data = sheet.data[0] + const rowData = data.rowData[row] + if (!rowData) { + return undefined + } + const cell = rowData.values[column] + if (!cell) { + return undefined + } + return cell + } + + private parseA1Notation(range: string): { + sheet: Sheet + topLeft: Range + bottomRight: Range + } { + let [sheetName, rest] = range.split("!") + const [topLeft, bottomRight] = rest.split(":") + + if (sheetName.startsWith("'") && sheetName.endsWith("'")) { + sheetName = sheetName.slice(1, -1) + } + + const sheet = this.getSheetByName(sheetName) + if (!sheet) { + throw new Error(`Sheet ${sheetName} not found`) + } + + return { + sheet, + topLeft: this.parseCell(topLeft), + bottomRight: this.parseCell(bottomRight), + } + } + + /** + * Parses a cell reference into a row and column. + * @param cell a string of the form A1, B2, etc. + * @returns + */ + private parseCell(cell: string): Range { + const firstChar = cell.slice(0, 1) + if (this.isInteger(firstChar)) { + return { row: parseInt(cell) - 1, column: "ALL" } + } + const column = this.letterToNumber(firstChar) + if (cell.length === 1) { + return { row: "ALL", column } + } + const number = cell.slice(1) + return { row: parseInt(number) - 1, column } + } + + private isInteger(value: string): boolean { + return !isNaN(parseInt(value)) + } + + private letterToNumber(letter: string): number { + return letter.charCodeAt(0) - 65 + } + + private getSheetByName(name: string): Sheet | undefined { + return this.sheet.sheets.find(sheet => sheet.properties.title === name) } } From 1bc84c16335eca43b637a6c0249d549f3e02b9d5 Mon Sep 17 00:00:00 2001 From: Sam Rose Date: Fri, 6 Sep 2024 17:29:56 +0100 Subject: [PATCH 46/85] 2nd test WIP. --- .../integrations/tests/googlesheets.spec.ts | 31 ++- .../integrations/tests/utils/googlesheets.ts | 210 ++++++++++++------ 2 files changed, 174 insertions(+), 67 deletions(-) diff --git a/packages/server/src/integrations/tests/googlesheets.spec.ts b/packages/server/src/integrations/tests/googlesheets.spec.ts index 21b7128999..25a3706e10 100644 --- a/packages/server/src/integrations/tests/googlesheets.spec.ts +++ b/packages/server/src/integrations/tests/googlesheets.spec.ts @@ -6,9 +6,9 @@ import { Datasource, FieldType, SourceName, + Table, TableSourceType, } from "@budibase/types" -import { access } from "node:fs" import { GoogleSheetsMock } from "./utils/googlesheets" describe("Google Sheets Integration", () => { @@ -84,4 +84,33 @@ describe("Google Sheets Integration", () => { expect(cell.userEnteredValue.stringValue).toEqual(table.name) }) }) + + describe("update", () => { + let table: Table + beforeEach(async () => { + table = await config.api.table.save({ + name: "Test Table", + type: "table", + sourceId: datasource._id!, + sourceType: TableSourceType.EXTERNAL, + schema: { + name: { + name: "name", + type: FieldType.STRING, + }, + description: { + name: "description", + type: FieldType.STRING, + }, + }, + }) + }) + + it.only("should be able to add a new row", async () => { + await config.api.row.save(table._id!, { + name: "Test Contact", + description: "original description", + }) + }) + }) }) diff --git a/packages/server/src/integrations/tests/utils/googlesheets.ts b/packages/server/src/integrations/tests/utils/googlesheets.ts index 72c7665679..242c18988c 100644 --- a/packages/server/src/integrations/tests/utils/googlesheets.ts +++ b/packages/server/src/integrations/tests/utils/googlesheets.ts @@ -6,8 +6,8 @@ type Value = string | number | boolean type Dimension = "ROWS" | "COLUMNS" interface Range { - row: number | "ALL" - column: number | "ALL" + row: number + column: number } interface DimensionProperties { @@ -124,7 +124,7 @@ interface Spreadsheet { export class GoogleSheetsMock { private config: GoogleSheetsConfig - private sheet: Spreadsheet + private spreadsheet: Spreadsheet static forDatasource(datasource: Datasource): GoogleSheetsMock { return new GoogleSheetsMock(datasource.config as GoogleSheetsConfig) @@ -132,7 +132,7 @@ export class GoogleSheetsMock { private constructor(config: GoogleSheetsConfig) { this.config = config - this.sheet = { + this.spreadsheet = { properties: { title: "Test Spreadsheet", }, @@ -142,11 +142,14 @@ export class GoogleSheetsMock { } init() { - nock("https://www.googleapis.com/").post("/oauth2/v4/token").reply(200, { - grant_type: "client_credentials", - client_id: "your-client-id", - client_secret: "your-client-secret", - }) + nock("https://www.googleapis.com/") + .post("/oauth2/v4/token") + .reply(200, { + grant_type: "client_credentials", + client_id: "your-client-id", + client_secret: "your-client-secret", + }) + .persist() nock("https://oauth2.googleapis.com/") .post("/token", { client_id: "test", @@ -160,54 +163,22 @@ export class GoogleSheetsMock { token_type: "Bearer", scopes: "https://www.googleapis.com/auth/spreadsheets", }) + .persist() nock("https://sheets.googleapis.com/", { reqheaders: { authorization: "Bearer test" }, }) - .get("/v4/spreadsheets/randomId/") - .reply(200, () => this.sheet) + .get(`/v4/spreadsheets/${this.config.spreadsheetId}/`) + .reply(200, () => this.handleGetSpreadsheet()) .persist() nock("https://sheets.googleapis.com/", { reqheaders: { authorization: "Bearer test" }, }) .post(`/v4/spreadsheets/${this.config.spreadsheetId}/:batchUpdate`) - .reply(200, (uri: string, request: nock.Body): nock.Body => { - const batchUpdateRequest = request as BatchUpdateRequest - const replies: Response[] = [] - - for (const request of batchUpdateRequest.requests) { - if (request.addSheet) { - const properties: SheetProperties = { - title: request.addSheet.properties.title, - sheetId: this.sheet.sheets.length, - gridProperties: { - rowCount: 100, - columnCount: 26, - frozenRowCount: 0, - frozenColumnCount: 0, - hideGridlines: false, - rowGroupControlAfter: false, - columnGroupControlAfter: false, - }, - } - - this.sheet.sheets.push({ - properties, - data: [this.createEmptyGrid(100, 26)], - }) - - replies.push({ addSheet: { properties } }) - } - } - - const response: BatchUpdateResponse = { - spreadsheetId: this.sheet.spreadsheetId, - replies, - updatedSpreadsheet: this.sheet, - } - return response - }) + .reply(200, (_uri, request) => + this.handleBatchUpdate(request as BatchUpdateRequest) + ) .persist() nock("https://sheets.googleapis.com/", { @@ -216,9 +187,94 @@ export class GoogleSheetsMock { .put( new RegExp(`/v4/spreadsheets/${this.config.spreadsheetId}/values/.*`) ) - .reply(200, (uri, request) => + .reply(200, (_uri, request) => this.handleValueUpdate(request as ValueRange) ) + .persist() + + nock("https://sheets.googleapis.com/", { + reqheaders: { authorization: "Bearer test" }, + }) + .get( + new RegExp(`/v4/spreadsheets/${this.config.spreadsheetId}/values/.*`) + ) + .reply(200, uri => { + const range = uri.split("/").pop() + if (!range) { + throw new Error("No range provided") + } + return this.handleGetValues(decodeURIComponent(range)) + }) + .persist() + } + + private handleGetValues(range: string): ValueRange { + const { sheet, topLeft, bottomRight } = this.parseA1Notation(range) + const valueRange: ValueRange = { + range, + majorDimension: "ROWS", + values: [], + } + + for (let row = topLeft.row; row <= bottomRight.row; row++) { + const values: Value[] = [] + for (let col = topLeft.column; col <= bottomRight.column; col++) { + const cell = this.getCellNumericIndexes(sheet, row, col) + if (!cell) { + throw new Error("Cell not found") + } + values.push(this.unwrapValue(cell.userEnteredValue)) + } + valueRange.values.push(values) + } + + return valueRange + } + + private handleBatchUpdate( + batchUpdateRequest: BatchUpdateRequest + ): BatchUpdateResponse { + const replies: Response[] = [] + + for (const request of batchUpdateRequest.requests) { + if (request.addSheet) { + const response = this.handleAddSheet(request.addSheet) + replies.push({ addSheet: response }) + } + } + + return { + spreadsheetId: this.spreadsheet.spreadsheetId, + replies, + updatedSpreadsheet: this.spreadsheet, + } + } + + private handleAddSheet(request: AddSheetRequest): AddSheetResponse { + const properties: SheetProperties = { + title: request.properties.title, + sheetId: this.spreadsheet.sheets.length, + gridProperties: { + rowCount: 100, + columnCount: 26, + frozenRowCount: 0, + frozenColumnCount: 0, + hideGridlines: false, + rowGroupControlAfter: false, + columnGroupControlAfter: false, + }, + } + + this.spreadsheet.sheets.push({ + properties, + data: [this.createEmptyGrid(100, 26)], + }) + + return { properties } + } + + private handleGetSpreadsheet(): Spreadsheet { + return this.spreadsheet } private handleValueUpdate(valueRange: ValueRange): UpdateValuesResponse { @@ -230,19 +286,6 @@ export class GoogleSheetsMock { valueRange.range ) - if (topLeft.row === "ALL") { - topLeft.row = 0 - } - if (bottomRight.row === "ALL") { - bottomRight.row = sheet.properties.gridProperties.rowCount - 1 - } - if (topLeft.column === "ALL") { - topLeft.column = 0 - } - if (bottomRight.column === "ALL") { - bottomRight.column = sheet.properties.gridProperties.columnCount - 1 - } - for (let row = topLeft.row; row <= bottomRight.row; row++) { for ( let column = topLeft.column; @@ -260,7 +303,7 @@ export class GoogleSheetsMock { } const response: UpdateValuesResponse = { - spreadsheetId: this.sheet.spreadsheetId, + spreadsheetId: this.spreadsheet.spreadsheetId, updatedRange: valueRange.range, updatedRows: valueRange.values.length, updatedColumns: valueRange.values[0].length, @@ -270,6 +313,20 @@ export class GoogleSheetsMock { return response } + private unwrapValue(from: ExtendedValue): Value { + if (from.stringValue !== undefined) { + return from.stringValue + } else if (from.numberValue !== undefined) { + return from.numberValue + } else if (from.boolValue !== undefined) { + return from.boolValue + } else if (from.formulaValue !== undefined) { + return from.formulaValue + } else { + throw new Error("Unsupported value type") + } + } + private createValue(from: Value): ExtendedValue { if (typeof from === "string") { return { @@ -375,10 +432,26 @@ export class GoogleSheetsMock { throw new Error(`Sheet ${sheetName} not found`) } + const parsedTopLeft = this.parseCell(topLeft) + const parsedBottomRight = this.parseCell(bottomRight) + + if (parsedTopLeft.row === "ALL") { + parsedTopLeft.row = 0 + } + if (parsedBottomRight.row === "ALL") { + parsedBottomRight.row = sheet.properties.gridProperties.rowCount - 1 + } + if (parsedTopLeft.column === "ALL") { + parsedTopLeft.column = 0 + } + if (parsedBottomRight.column === "ALL") { + parsedBottomRight.column = sheet.properties.gridProperties.columnCount - 1 + } + return { sheet, - topLeft: this.parseCell(topLeft), - bottomRight: this.parseCell(bottomRight), + topLeft: parsedTopLeft as Range, + bottomRight: parsedBottomRight as Range, } } @@ -387,7 +460,10 @@ export class GoogleSheetsMock { * @param cell a string of the form A1, B2, etc. * @returns */ - private parseCell(cell: string): Range { + private parseCell(cell: string): { + row: number | "ALL" + column: number | "ALL" + } { const firstChar = cell.slice(0, 1) if (this.isInteger(firstChar)) { return { row: parseInt(cell) - 1, column: "ALL" } @@ -409,6 +485,8 @@ export class GoogleSheetsMock { } private getSheetByName(name: string): Sheet | undefined { - return this.sheet.sheets.find(sheet => sheet.properties.title === name) + return this.spreadsheet.sheets.find( + sheet => sheet.properties.title === name + ) } } From 1c5b50773f8e16131733611c0615193e0889b0c8 Mon Sep 17 00:00:00 2001 From: Sam Rose Date: Mon, 9 Sep 2024 12:05:15 +0100 Subject: [PATCH 47/85] Docs to Google Sheets mock. --- .../integrations/tests/utils/googlesheets.ts | 178 ++++++++++++++---- 1 file changed, 144 insertions(+), 34 deletions(-) diff --git a/packages/server/src/integrations/tests/utils/googlesheets.ts b/packages/server/src/integrations/tests/utils/googlesheets.ts index 242c18988c..82a274405b 100644 --- a/packages/server/src/integrations/tests/utils/googlesheets.ts +++ b/packages/server/src/integrations/tests/utils/googlesheets.ts @@ -2,7 +2,10 @@ import { Datasource } from "@budibase/types" import nock from "nock" import { GoogleSheetsConfig } from "../../googlesheets" +// https://protobuf.dev/reference/protobuf/google.protobuf/#value type Value = string | number | boolean + +// https://developers.google.com/sheets/api/reference/rest/v4/Dimension type Dimension = "ROWS" | "COLUMNS" interface Range { @@ -18,12 +21,14 @@ interface DimensionProperties { // dataSourceColumnReference: DataSourceColumnReference } +// https://developers.google.com/sheets/api/reference/rest/v4/spreadsheets.values#ValueRange interface ValueRange { range: string majorDimension: Dimension values: Value[][] } +// https://developers.google.com/sheets/api/reference/rest/v4/UpdateValuesResponse interface UpdateValuesResponse { spreadsheetId: string updatedRange: string @@ -33,6 +38,7 @@ interface UpdateValuesResponse { updatedData: ValueRange } +// https://developers.google.com/sheets/api/reference/rest/v4/spreadsheets/response#AddSheetResponse interface AddSheetResponse { properties: SheetProperties } @@ -41,12 +47,14 @@ interface Response { addSheet?: AddSheetResponse } +// https://developers.google.com/sheets/api/reference/rest/v4/spreadsheets/response interface BatchUpdateResponse { spreadsheetId: string replies: Response[] updatedSpreadsheet: Spreadsheet } +// https://developers.google.com/sheets/api/reference/rest/v4/spreadsheets/sheets#GridProperties interface GridProperties { rowCount: number columnCount: number @@ -57,12 +65,14 @@ interface GridProperties { columnGroupControlAfter: boolean } +// https://developers.google.com/sheets/api/reference/rest/v4/spreadsheets/sheets#SheetProperties interface SheetProperties { sheetId: number title: string gridProperties: GridProperties } +// https://developers.google.com/sheets/api/reference/rest/v4/spreadsheets/request#AddSheetRequest interface AddSheetRequest { properties: SheetProperties } @@ -71,6 +81,7 @@ interface Request { addSheet?: AddSheetRequest } +// https://developers.google.com/sheets/api/reference/rest/v4/spreadsheets/request interface BatchUpdateRequest { requests: Request[] includeSpreadsheetInResponse: boolean @@ -78,11 +89,13 @@ interface BatchUpdateRequest { responseIncludeGridData: boolean } +// https://developers.google.com/sheets/api/reference/rest/v4/spreadsheets/other#ErrorValue interface ErrorValue { type: string message: string } +// https://developers.google.com/sheets/api/reference/rest/v4/spreadsheets/other#ExtendedValue interface ExtendedValue { stringValue?: string numberValue?: number @@ -91,14 +104,17 @@ interface ExtendedValue { errorValue?: ErrorValue } +// https://developers.google.com/sheets/api/reference/rest/v4/spreadsheets/cells#CellData interface CellData { userEnteredValue: ExtendedValue } +// https://developers.google.com/sheets/api/reference/rest/v4/spreadsheets/sheets#RowData interface RowData { values: CellData[] } +// https://developers.google.com/sheets/api/reference/rest/v4/spreadsheets/sheets#GridData interface GridData { startRow: number startColumn: number @@ -107,21 +123,61 @@ interface GridData { columnMetadata: DimensionProperties[] } +// https://developers.google.com/sheets/api/reference/rest/v4/spreadsheets/sheets#Sheet interface Sheet { properties: SheetProperties data: GridData[] } +// https://developers.google.com/sheets/api/reference/rest/v4/spreadsheets#SpreadsheetProperties interface SpreadsheetProperties { title: string } +// https://developers.google.com/sheets/api/reference/rest/v4/spreadsheets#Spreadsheet interface Spreadsheet { properties: SpreadsheetProperties spreadsheetId: string sheets: Sheet[] } +// https://developers.google.com/sheets/api/reference/rest/v4/ValueInputOption +type ValueInputOption = + | "USER_ENTERED" + | "RAW" + | "INPUT_VALUE_OPTION_UNSPECIFIED" + +// https://developers.google.com/sheets/api/reference/rest/v4/spreadsheets.values/append#InsertDataOption +type InsertDataOption = "OVERWRITE" | "INSERT_ROWS" + +// https://developers.google.com/sheets/api/reference/rest/v4/ValueRenderOption +type ValueRenderOption = "FORMATTED_VALUE" | "UNFORMATTED_VALUE" | "FORMULA" + +// https://developers.google.com/sheets/api/reference/rest/v4/DateTimeRenderOption +type DateTimeRenderOption = "SERIAL_NUMBER" | "FORMATTED_STRING" + +// https://developers.google.com/sheets/api/reference/rest/v4/spreadsheets.values/append#query-parameters +interface AppendParams { + valueInputOption?: ValueInputOption + insertDataOption?: InsertDataOption + includeValuesInResponse?: boolean + responseValueRenderOption?: ValueRenderOption + responseDateTimeRenderOption?: DateTimeRenderOption +} + +interface AppendRequest { + range: string + params: AppendParams + body: ValueRange +} + +// https://developers.google.com/sheets/api/reference/rest/v4/spreadsheets.values/append#response-body +interface AppendResponse { + spreadsheetId: string + tableRange: string + updates: UpdateValuesResponse +} + export class GoogleSheetsMock { private config: GoogleSheetsConfig private spreadsheet: Spreadsheet @@ -141,7 +197,38 @@ export class GoogleSheetsMock { } } - init() { + private route( + method: "get" | "put" | "post", + path: string | RegExp, + handler: (uri: string, request: nock.Body) => nock.Body + ): nock.Scope { + const headers = { reqheaders: { authorization: "Bearer test" } } + const scope = nock("https://sheets.googleapis.com/", headers) + return scope[method](path).reply(200, handler).persist() + } + + private get( + path: string | RegExp, + handler: (uri: string, request: nock.Body) => nock.Body + ): nock.Scope { + return this.route("get", path, handler) + } + + private put( + path: string | RegExp, + handler: (uri: string, request: nock.Body) => nock.Body + ): nock.Scope { + return this.route("put", path, handler) + } + + private post( + path: string | RegExp, + handler: (uri: string, request: nock.Body) => nock.Body + ): nock.Scope { + return this.route("post", path, handler) + } + + private mockAuth() { nock("https://www.googleapis.com/") .post("/oauth2/v4/token") .reply(200, { @@ -164,50 +251,72 @@ export class GoogleSheetsMock { scopes: "https://www.googleapis.com/auth/spreadsheets", }) .persist() + } - nock("https://sheets.googleapis.com/", { - reqheaders: { authorization: "Bearer test" }, - }) - .get(`/v4/spreadsheets/${this.config.spreadsheetId}/`) - .reply(200, () => this.handleGetSpreadsheet()) - .persist() + init() { + this.mockAuth() - nock("https://sheets.googleapis.com/", { - reqheaders: { authorization: "Bearer test" }, - }) - .post(`/v4/spreadsheets/${this.config.spreadsheetId}/:batchUpdate`) - .reply(200, (_uri, request) => - this.handleBatchUpdate(request as BatchUpdateRequest) - ) - .persist() + this.get(`/v4/spreadsheets/${this.config.spreadsheetId}/`, () => + this.handleGetSpreadsheet() + ) - nock("https://sheets.googleapis.com/", { - reqheaders: { authorization: "Bearer test" }, - }) - .put( - new RegExp(`/v4/spreadsheets/${this.config.spreadsheetId}/values/.*`) - ) - .reply(200, (_uri, request) => - this.handleValueUpdate(request as ValueRange) - ) - .persist() + // https://developers.google.com/sheets/api/reference/rest/v4/spreadsheets.values/batchUpdate + this.post( + `/v4/spreadsheets/${this.config.spreadsheetId}/:batchUpdate`, + (_uri, request) => this.handleBatchUpdate(request as BatchUpdateRequest) + ) - nock("https://sheets.googleapis.com/", { - reqheaders: { authorization: "Bearer test" }, - }) - .get( - new RegExp(`/v4/spreadsheets/${this.config.spreadsheetId}/values/.*`) - ) - .reply(200, uri => { + // https://developers.google.com/sheets/api/reference/rest/v4/spreadsheets.values/update + this.put( + new RegExp(`/v4/spreadsheets/${this.config.spreadsheetId}/values/.*`), + (_uri, request) => this.handleValueUpdate(request as ValueRange) + ) + + // https://developers.google.com/sheets/api/reference/rest/v4/spreadsheets.values/get + this.get( + new RegExp(`/v4/spreadsheets/${this.config.spreadsheetId}/values/.*`), + uri => { const range = uri.split("/").pop() if (!range) { throw new Error("No range provided") } return this.handleGetValues(decodeURIComponent(range)) - }) - .persist() + } + ) + + // https://developers.google.com/sheets/api/reference/rest/v4/spreadsheets.values/append + this.post( + new RegExp( + `/v4/spreadsheets/${this.config.spreadsheetId}/values/.*:append` + ), + (_uri, request) => { + const url = new URL(_uri, "https://sheets.googleapis.com/") + const params: Record = Object.fromEntries( + url.searchParams.entries() + ) + + if (params.includeValuesInResponse === "true") { + params.includeValuesInResponse = true + } else { + params.includeValuesInResponse = false + } + + const range = url.pathname.split("/").pop() + if (!range) { + throw new Error("No range provided") + } + + return this.handleValueAppend({ + range, + params, + body: request as ValueRange, + }) + } + ) } + private handleValueAppend(request: AppendRequest): AppendResponse {} + private handleGetValues(range: string): ValueRange { const { sheet, topLeft, bottomRight } = this.parseA1Notation(range) const valueRange: ValueRange = { @@ -415,6 +524,7 @@ export class GoogleSheetsMock { return cell } + // https://developers.google.com/sheets/api/guides/concepts#cell private parseA1Notation(range: string): { sheet: Sheet topLeft: Range From 57698170a8d2983b3396919aa6c3266f8835a0e5 Mon Sep 17 00:00:00 2001 From: Martin McKeaveney Date: Mon, 9 Sep 2024 13:55:26 +0100 Subject: [PATCH 48/85] lint --- packages/backend-core/src/configs/configs.ts | 2 - .../core/utilities/structures/licenses.ts | 2 +- .../portal/settings/ai/AIConfigTile.svelte | 8 +- .../portal/settings/ai/AISettings.spec.js | 18 +- .../portal/settings/ai/ConfigModal.svelte | 27 +-- .../builder/portal/settings/ai/constants.ts | 49 ++-- .../builder/portal/settings/ai/index.svelte | 14 +- .../portal/settings/ai/logos/Anthropic.svelte | 18 +- .../portal/settings/ai/logos/Budibase.svelte | 31 ++- .../portal/settings/ai/logos/OpenAI.svelte | 15 +- .../settings/ai/logos/TogetherAI.svelte | 66 ++++- .../builder/portal/users/users/index.svelte | 2 +- packages/builder/vitest.setup.js | 2 +- packages/types/src/documents/global/config.ts | 2 +- packages/types/src/sdk/licensing/feature.ts | 2 +- packages/types/src/sdk/licensing/quota.ts | 2 +- .../src/api/controllers/global/configs.ts | 5 +- .../controllers/global/tests/configs.spec.ts | 44 ++-- yarn.lock | 228 ++++++++---------- 19 files changed, 293 insertions(+), 244 deletions(-) diff --git a/packages/backend-core/src/configs/configs.ts b/packages/backend-core/src/configs/configs.ts index 1eafd9596e..0d189e3f7d 100644 --- a/packages/backend-core/src/configs/configs.ts +++ b/packages/backend-core/src/configs/configs.ts @@ -1,6 +1,4 @@ import { - AIConfig, - AIInnerConfig, Config, ConfigType, GoogleConfig, diff --git a/packages/backend-core/tests/core/utilities/structures/licenses.ts b/packages/backend-core/tests/core/utilities/structures/licenses.ts index cf422637d6..b4d209a7ee 100644 --- a/packages/backend-core/tests/core/utilities/structures/licenses.ts +++ b/packages/backend-core/tests/core/utilities/structures/licenses.ts @@ -59,7 +59,7 @@ export function quotas(): Quotas { name: "Budibase AI Credits", value: 1, triggers: [], - } + }, }, static: { rows: { diff --git a/packages/builder/src/pages/builder/portal/settings/ai/AIConfigTile.svelte b/packages/builder/src/pages/builder/portal/settings/ai/AIConfigTile.svelte index 0b7a006312..d4d5279d3e 100644 --- a/packages/builder/src/pages/builder/portal/settings/ai/AIConfigTile.svelte +++ b/packages/builder/src/pages/builder/portal/settings/ai/AIConfigTile.svelte @@ -18,13 +18,13 @@
{#if config.name === "Budibase AI"} - + {:else if config.provider === Providers.OpenAI.name} - + {:else if config.provider === Providers.Anthropic.name} - + {:else if config.provider === Providers.TogetherAI.name} - + {/if}
diff --git a/packages/builder/src/pages/builder/portal/settings/ai/AISettings.spec.js b/packages/builder/src/pages/builder/portal/settings/ai/AISettings.spec.js index 5ca14c7898..805739957b 100644 --- a/packages/builder/src/pages/builder/portal/settings/ai/AISettings.spec.js +++ b/packages/builder/src/pages/builder/portal/settings/ai/AISettings.spec.js @@ -11,23 +11,19 @@ const Hosting = { Self: "self", } -function setupEnv( - hosting, - features = {}) { +function setupEnv(hosting, features = {}) { const defaultFeatures = { budibaseAIEnabled: false, customAIConfigsEnabled: false, - ...features + ...features, } admin.subscribe = vi.fn().mockImplementation(callback => { - callback({cloud: hosting === Hosting.Cloud}) - return () => { - } + callback({ cloud: hosting === Hosting.Cloud }) + return () => {} }) licensing.subscribe = vi.fn().mockImplementation(callback => { callback(defaultFeatures) - return () => { - } + return () => {} }) } @@ -73,7 +69,7 @@ describe("AISettings", () => { addConfigurationButton = instance.queryByText("Add configuration") expect(addConfigurationButton).not.toBeInTheDocument() - setupEnv(Hosting.Cloud, {customAIConfigsEnabled: true}) + setupEnv(Hosting.Cloud, { customAIConfigsEnabled: true }) instance = render(AISettings) addConfigurationButton = instance.queryByText("Add configuration") expect(addConfigurationButton).toBeInTheDocument() @@ -87,7 +83,7 @@ describe("AISettings", () => { addConfigurationButton = instance.queryByText("Add configuration") expect(addConfigurationButton).not.toBeInTheDocument() - setupEnv(Hosting.Self, {customAIConfigsEnabled: true}) + setupEnv(Hosting.Self, { customAIConfigsEnabled: true }) instance = render(AISettings, {}) addConfigurationButton = instance.queryByText("Add configuration") expect(addConfigurationButton).toBeInTheDocument() diff --git a/packages/builder/src/pages/builder/portal/settings/ai/ConfigModal.svelte b/packages/builder/src/pages/builder/portal/settings/ai/ConfigModal.svelte index 99f3aa8e7b..2f7486d087 100644 --- a/packages/builder/src/pages/builder/portal/settings/ai/ConfigModal.svelte +++ b/packages/builder/src/pages/builder/portal/settings/ai/ConfigModal.svelte @@ -1,12 +1,6 @@ - + + - - \ No newline at end of file + + diff --git a/packages/builder/src/pages/builder/portal/settings/ai/logos/Budibase.svelte b/packages/builder/src/pages/builder/portal/settings/ai/logos/Budibase.svelte index f69c7d8281..dc19facb1a 100644 --- a/packages/builder/src/pages/builder/portal/settings/ai/logos/Budibase.svelte +++ b/packages/builder/src/pages/builder/portal/settings/ai/logos/Budibase.svelte @@ -2,16 +2,35 @@ export let width export let height - + + - - - - + + + + - + diff --git a/packages/builder/src/pages/builder/portal/settings/ai/logos/OpenAI.svelte b/packages/builder/src/pages/builder/portal/settings/ai/logos/OpenAI.svelte index fa60b33cfd..a049cd537f 100644 --- a/packages/builder/src/pages/builder/portal/settings/ai/logos/OpenAI.svelte +++ b/packages/builder/src/pages/builder/portal/settings/ai/logos/OpenAI.svelte @@ -2,8 +2,17 @@ export let height export let width - + + OpenAI icon - \ No newline at end of file + d="M22.2819 9.8211a5.9847 5.9847 0 0 0-.5157-4.9108 6.0462 6.0462 0 0 0-6.5098-2.9A6.0651 6.0651 0 0 0 4.9807 4.1818a5.9847 5.9847 0 0 0-3.9977 2.9 6.0462 6.0462 0 0 0 .7427 7.0966 5.98 5.98 0 0 0 .511 4.9107 6.051 6.051 0 0 0 6.5146 2.9001A5.9847 5.9847 0 0 0 13.2599 24a6.0557 6.0557 0 0 0 5.7718-4.2058 5.9894 5.9894 0 0 0 3.9977-2.9001 6.0557 6.0557 0 0 0-.7475-7.0729zm-9.022 12.6081a4.4755 4.4755 0 0 1-2.8764-1.0408l.1419-.0804 4.7783-2.7582a.7948.7948 0 0 0 .3927-.6813v-6.7369l2.02 1.1686a.071.071 0 0 1 .038.052v5.5826a4.504 4.504 0 0 1-4.4945 4.4944zm-9.6607-4.1254a4.4708 4.4708 0 0 1-.5346-3.0137l.142.0852 4.783 2.7582a.7712.7712 0 0 0 .7806 0l5.8428-3.3685v2.3324a.0804.0804 0 0 1-.0332.0615L9.74 19.9502a4.4992 4.4992 0 0 1-6.1408-1.6464zM2.3408 7.8956a4.485 4.485 0 0 1 2.3655-1.9728V11.6a.7664.7664 0 0 0 .3879.6765l5.8144 3.3543-2.0201 1.1685a.0757.0757 0 0 1-.071 0l-4.8303-2.7865A4.504 4.504 0 0 1 2.3408 7.872zm16.5963 3.8558L13.1038 8.364 15.1192 7.2a.0757.0757 0 0 1 .071 0l4.8303 2.7913a4.4944 4.4944 0 0 1-.6765 8.1042v-5.6772a.79.79 0 0 0-.407-.667zm2.0107-3.0231l-.142-.0852-4.7735-2.7818a.7759.7759 0 0 0-.7854 0L9.409 9.2297V6.8974a.0662.0662 0 0 1 .0284-.0615l4.8303-2.7866a4.4992 4.4992 0 0 1 6.6802 4.66zM8.3065 12.863l-2.02-1.1638a.0804.0804 0 0 1-.038-.0567V6.0742a4.4992 4.4992 0 0 1 7.3757-3.4537l-.142.0805L8.704 5.459a.7948.7948 0 0 0-.3927.6813zm1.0976-2.3654l2.602-1.4998 2.6069 1.4998v2.9994l-2.5974 1.4997-2.6067-1.4997Z" + /> + diff --git a/packages/builder/src/pages/builder/portal/settings/ai/logos/TogetherAI.svelte b/packages/builder/src/pages/builder/portal/settings/ai/logos/TogetherAI.svelte index 1afd5626b1..b81450f768 100644 --- a/packages/builder/src/pages/builder/portal/settings/ai/logos/TogetherAI.svelte +++ b/packages/builder/src/pages/builder/portal/settings/ai/logos/TogetherAI.svelte @@ -2,16 +2,56 @@ export let height export let width - - - - - - - - - - - - - \ No newline at end of file + + + + + + + + + + + + + + diff --git a/packages/builder/src/pages/builder/portal/users/users/index.svelte b/packages/builder/src/pages/builder/portal/users/users/index.svelte index b91b2129e6..91d9c99af6 100644 --- a/packages/builder/src/pages/builder/portal/users/users/index.svelte +++ b/packages/builder/src/pages/builder/portal/users/users/index.svelte @@ -127,7 +127,7 @@ name: user.firstName ? user.firstName + " " + user.lastName : "", userGroups, __selectable: - role.value === Constants.BudibaseRoles.Owner ? false : undefined, + role?.value === Constants.BudibaseRoles.Owner ? false : undefined, apps: [...new Set(Object.keys(user.roles))], access: role.sortOrder, } diff --git a/packages/builder/vitest.setup.js b/packages/builder/vitest.setup.js index 92df5e9600..0204a1a772 100644 --- a/packages/builder/vitest.setup.js +++ b/packages/builder/vitest.setup.js @@ -1,4 +1,4 @@ -import { expect, vi } from "vitest" +import { expect } from "vitest" import "@testing-library/jest-dom/vitest" global.ResizeObserver = require("resize-observer-polyfill") diff --git a/packages/types/src/documents/global/config.ts b/packages/types/src/documents/global/config.ts index 2b0b97ccc0..6e9c29af97 100644 --- a/packages/types/src/documents/global/config.ts +++ b/packages/types/src/documents/global/config.ts @@ -153,5 +153,5 @@ export enum ConfigType { OIDC = "oidc", OIDC_LOGOS = "logos_oidc", SCIM = "scim", - AI = "ai" + AI = "ai", } diff --git a/packages/types/src/sdk/licensing/feature.ts b/packages/types/src/sdk/licensing/feature.ts index bf2bcb72b6..cbcb16f7b6 100644 --- a/packages/types/src/sdk/licensing/feature.ts +++ b/packages/types/src/sdk/licensing/feature.ts @@ -16,7 +16,7 @@ export enum Feature { VIEW_PERMISSIONS = "viewPermissions", VIEW_READONLY_COLUMNS = "viewReadonlyColumns", BUDIBASE_AI = "budibaseAI", - AI_CUSTOM_CONFIGS = "customAIConfigs" + AI_CUSTOM_CONFIGS = "customAIConfigs", } export type PlanFeatures = { [key in PlanType]: Feature[] | undefined } diff --git a/packages/types/src/sdk/licensing/quota.ts b/packages/types/src/sdk/licensing/quota.ts index 748d90fda7..75fc18e8e6 100644 --- a/packages/types/src/sdk/licensing/quota.ts +++ b/packages/types/src/sdk/licensing/quota.ts @@ -24,7 +24,7 @@ export enum MonthlyQuotaName { QUERIES = "queries", AUTOMATIONS = "automations", DAY_PASSES = "dayPasses", - BUDIBASE_AI_CREDITS = "budibaseAICredits" + BUDIBASE_AI_CREDITS = "budibaseAICredits", } export enum ConstantQuotaName { diff --git a/packages/worker/src/api/controllers/global/configs.ts b/packages/worker/src/api/controllers/global/configs.ts index 4a19168c4f..d3eef71240 100644 --- a/packages/worker/src/api/controllers/global/configs.ts +++ b/packages/worker/src/api/controllers/global/configs.ts @@ -199,7 +199,10 @@ async function verifyOIDCConfig(config: OIDCConfigs) { await verifySSOConfig(ConfigType.OIDC, config.configs[0]) } -export async function verifyAIConfig(config: AIConfig, existingConfig?: AIConfig) { +export async function verifyAIConfig( + config: AIConfig, + existingConfig?: AIConfig +) { if (!existingConfig) return // ensure that the redacted API keys are not overwritten in the DB diff --git a/packages/worker/src/api/controllers/global/tests/configs.spec.ts b/packages/worker/src/api/controllers/global/tests/configs.spec.ts index 393a11a043..2ef00252ff 100644 --- a/packages/worker/src/api/controllers/global/tests/configs.spec.ts +++ b/packages/worker/src/api/controllers/global/tests/configs.spec.ts @@ -1,3 +1,4 @@ +import { expect } from "vitest" import { configs } from "@budibase/backend-core" import { UserCtx } from "@budibase/types" import * as pro from "@budibase/pro" @@ -7,11 +8,10 @@ jest.mock("@budibase/backend-core", () => ({ ...jest.requireActual("@budibase/backend-core"), configs: { getConfig: jest.fn(), - save: jest.fn() + save: jest.fn(), }, })) - describe("Global configs controller", () => { afterEach(() => { jest.resetAllMocks() @@ -23,14 +23,14 @@ describe("Global configs controller", () => { ai: { apiKey: "abc123APIKey", baseUrl: "https://api.example.com", - } - } + }, + }, }) const ctx = { params: { - type: "ai" + type: "ai", }, - throw: jest.fn() + throw: jest.fn(), } as UserCtx await find(ctx) @@ -39,9 +39,9 @@ describe("Global configs controller", () => { config: { ai: { apiKey: "--secret-value--", - "baseUrl": "https://api.example.com" - } - } + baseUrl: "https://api.example.com", + }, + }, }) }) @@ -52,14 +52,14 @@ describe("Global configs controller", () => { ai: { apiKey: "abc123APIKey", baseUrl: "https://api.example.com", - } - } + }, + }, }) const ctx = { params: { - type: "ai" + type: "ai", }, - throw: jest.fn() + throw: jest.fn(), } as UserCtx await find(ctx) @@ -75,9 +75,9 @@ describe("Global configs controller", () => { }, ai: { apiKey: "--secret-value--", - "baseUrl": "https://api.example.com" - } - } + baseUrl: "https://api.example.com", + }, + }, }) }) @@ -91,9 +91,9 @@ describe("Global configs controller", () => { name: "MyConfig", active: true, defaultModel: "gpt4", - apiKey: "--secret-value--" - } - } + apiKey: "--secret-value--", + }, + }, } const existingConfig = { @@ -105,9 +105,9 @@ describe("Global configs controller", () => { name: "MyConfig", active: true, defaultModel: "gpt4", - apiKey: "myapikey" - } - } + apiKey: "myapikey", + }, + }, } await verifyAIConfig(newConfig, existingConfig) diff --git a/yarn.lock b/yarn.lock index 4e4e528334..146fa74d5b 100644 --- a/yarn.lock +++ b/yarn.lock @@ -2053,6 +2053,44 @@ resolved "https://registry.yarnpkg.com/@bcoe/v8-coverage/-/v8-coverage-0.2.3.tgz#75a2e8b51cb758a7553d6804a5932d7aace75c39" integrity sha512-0hYQ8SB4Db5zvZB4axdMHGwEaQjkZzFjQiN9LVYvIFB2nSUHW9tYpxWriPrWDASIxiaXax83REcLxuSdnGPZtw== +"@budibase/backend-core@2.31.8": + version "0.0.0" + dependencies: + "@budibase/nano" "10.1.5" + "@budibase/pouchdb-replication-stream" "1.2.11" + "@budibase/shared-core" "0.0.0" + "@budibase/types" "0.0.0" + aws-cloudfront-sign "3.0.2" + aws-sdk "2.1030.0" + bcrypt "5.1.0" + bcryptjs "2.4.3" + bull "4.10.1" + correlation-id "4.0.0" + dd-trace "5.2.0" + dotenv "16.0.1" + ioredis "5.3.2" + joi "17.6.0" + jsonwebtoken "9.0.2" + knex "2.4.2" + koa-passport "^6.0.0" + koa-pino-logger "4.0.0" + lodash "4.17.21" + node-fetch "2.6.7" + passport-google-oauth "2.0.0" + passport-local "1.0.0" + passport-oauth2-refresh "^2.1.0" + pino "8.11.0" + pino-http "8.3.3" + posthog-node "4.0.1" + pouchdb "7.3.0" + pouchdb-find "7.2.2" + redlock "4.2.0" + rotating-file-stream "3.1.0" + sanitize-s3-objectkey "0.0.1" + semver "^7.5.4" + tar-fs "2.1.1" + uuid "^8.3.2" + "@budibase/handlebars-helpers@^0.13.2": version "0.13.2" resolved "https://registry.yarnpkg.com/@budibase/handlebars-helpers/-/handlebars-helpers-0.13.2.tgz#73ab51c464e91fd955b429017648e0257060db77" @@ -2095,6 +2133,45 @@ pouchdb-promise "^6.0.4" through2 "^2.0.0" +"@budibase/pro@npm:@budibase/pro@latest": + version "2.31.8" + resolved "https://registry.yarnpkg.com/@budibase/pro/-/pro-2.31.8.tgz#92b27f99f815f5d20bf58bfae916760b14a036da" + integrity sha512-nmNKVoMdUVqEIq6xqoBq0gVBCLkoPMszmn0Zu0SJ/Dc2SpsXhPz9S3n9xXfAA+FHUg9LgUAS+eKPCKPWZXtDHQ== + dependencies: + "@budibase/backend-core" "2.31.8" + "@budibase/shared-core" "2.31.8" + "@budibase/string-templates" "2.31.8" + "@budibase/types" "2.31.8" + "@koa/router" "8.0.8" + bull "4.10.1" + dd-trace "5.2.0" + joi "17.6.0" + jsonwebtoken "9.0.2" + lru-cache "^7.14.1" + memorystream "^0.3.1" + node-fetch "2.6.7" + scim-patch "^0.8.1" + scim2-parse-filter "^0.2.8" + +"@budibase/shared-core@2.31.8": + version "0.0.0" + dependencies: + "@budibase/types" "0.0.0" + cron-validate "1.4.5" + +"@budibase/string-templates@2.31.8": + version "0.0.0" + dependencies: + "@budibase/handlebars-helpers" "^0.13.2" + dayjs "^1.10.8" + handlebars "^4.7.8" + lodash.clonedeep "^4.5.0" + +"@budibase/types@2.31.8": + version "0.0.0" + dependencies: + scim-patch "^0.8.1" + "@bull-board/api@5.10.2": version "5.10.2" resolved "https://registry.yarnpkg.com/@bull-board/api/-/api-5.10.2.tgz#ae8ff6918b23897bf879a6ead3683f964374c4b3" @@ -7448,30 +7525,7 @@ axios-retry@^3.1.9: "@babel/runtime" "^7.15.4" is-retry-allowed "^2.2.0" -axios@0.24.0: - version "0.24.0" - resolved "https://registry.yarnpkg.com/axios/-/axios-0.24.0.tgz#804e6fa1e4b9c5288501dd9dff56a7a0940d20d6" - integrity sha512-Q6cWsys88HoPgAaFAVUb0WpPk0O8iTeisR9IMqy9G8AbO4NlpVknrnQS03zzF9PGAWgO3cgletO3VjV/P7VztA== - dependencies: - follow-redirects "^1.14.4" - -axios@1.1.3: - version "1.1.3" - resolved "https://registry.yarnpkg.com/axios/-/axios-1.1.3.tgz#8274250dada2edf53814ed7db644b9c2866c1e35" - integrity sha512-00tXVRwKx/FZr/IDVFt4C+f9FYairX517WoGCL6dpOntqLkZofjhu43F/Xl44UOpqa+9sLFDrG/XAnFsUYgkDA== - dependencies: - follow-redirects "^1.15.0" - form-data "^4.0.0" - proxy-from-env "^1.1.0" - -axios@^0.21.1: - version "0.21.4" - resolved "https://registry.yarnpkg.com/axios/-/axios-0.21.4.tgz#c67b90dc0568e5c1cf2b0b858c43ba28e2eda575" - integrity sha512-ut5vewkiu8jjGBdqpM44XxjuCjq9LAKeHVmoVfHVzy8eHgxxq8SbAVQNovDA8mVi05kP0Ea/n/UzcSHcTJQfNg== - dependencies: - follow-redirects "^1.14.0" - -axios@^1.0.0, axios@^1.1.3, axios@^1.4.0, axios@^1.5.0, axios@^1.6.2: +axios@0.24.0, axios@1.1.3, axios@1.6.3, axios@^0.21.1, axios@^1.0.0, axios@^1.1.3, axios@^1.4.0, axios@^1.5.0, axios@^1.6.2: version "1.6.3" resolved "https://registry.yarnpkg.com/axios/-/axios-1.6.3.tgz#7f50f23b3aa246eff43c54834272346c396613f4" integrity sha512-fWyNdeawGam70jXSVlKl+SUNVcL6j6W79CuSIPfi6HnDUmSCH6gyUys/HrqHeA/wU0Az41rRgean494d0Jb+ww== @@ -11581,11 +11635,6 @@ fn.name@1.x.x: resolved "https://registry.yarnpkg.com/fn.name/-/fn.name-1.1.0.tgz#26cad8017967aea8731bc42961d04a3d5988accc" integrity sha512-GRnmB5gPyJpAhTQdSZTSp9uaPSvl09KoYcMQtsB9rQoOmzs9dH6ffeccH+Z+cv6P68Hu5bC6JjRh4Ah/mHSNRw== -follow-redirects@^1.14.0, follow-redirects@^1.14.4: - version "1.15.9" - resolved "https://registry.yarnpkg.com/follow-redirects/-/follow-redirects-1.15.9.tgz#a604fa10e443bf98ca94228d9eebcc2e8a2c8ee1" - integrity sha512-gew4GsXizNgdoRyqmyfMHyAmXsZDk6mHkSxZFCzW9gwlbtOW44CDtYavM+y+72qD/Vq2l550kMF52DT8fOLJqQ== - follow-redirects@^1.15.0: version "1.15.6" resolved "https://registry.yarnpkg.com/follow-redirects/-/follow-redirects-1.15.6.tgz#7f815c0cda4249c74ff09e95ef97c23b5fd0399b" @@ -12679,12 +12728,7 @@ http-assert@^1.3.0: deep-equal "~1.0.1" http-errors "~1.8.0" -http-cache-semantics@3.8.1: - version "3.8.1" - resolved "https://registry.yarnpkg.com/http-cache-semantics/-/http-cache-semantics-3.8.1.tgz#39b0e16add9b605bf0a9ef3d9daaf4843b4cacd2" - integrity sha512-5ai2iksyV8ZXmnZhHH4rWPoxxistEexSi5936zIQ1bnNTW5VnA85B6P/VpXiRM017IgRvb2kKo1a//y+0wSp3w== - -http-cache-semantics@^4.0.0, http-cache-semantics@^4.1.0, http-cache-semantics@^4.1.1: +http-cache-semantics@3.8.1, http-cache-semantics@4.1.1, http-cache-semantics@^4.0.0, http-cache-semantics@^4.1.0, http-cache-semantics@^4.1.1: version "4.1.1" resolved "https://registry.yarnpkg.com/http-cache-semantics/-/http-cache-semantics-4.1.1.tgz#abe02fcb2985460bf0323be664436ec3476a6d5a" integrity sha512-er295DKPVsV82j5kw1Gjt+ADA/XYHsajl82cGNQG2eyoPkvgUhX+nDIyelzhIWbbsXP39EHcI6l5tYs2FYqYXQ== @@ -13695,6 +13739,11 @@ isobject@^3.0.1: resolved "https://registry.yarnpkg.com/isobject/-/isobject-3.0.1.tgz#4e431e92b11a9731636aa1f9c8d1ccbcfdab78df" integrity sha512-WhB9zCku7EGTj/HQQRz5aUQEUeoQZH2bWcltRErOpymJ4boYE6wL9Tbr23krRPSZ+C5zqNSrSw+Cc7sZZ4b7vg== +isobject@^4.0.0: + version "4.0.0" + resolved "https://registry.yarnpkg.com/isobject/-/isobject-4.0.0.tgz#3f1c9155e73b192022a80819bacd0343711697b0" + integrity sha512-S/2fF5wH8SJA/kmwr6HYhK/RI/OkhD84k8ntalo0iJjZikgq1XFvR5M8NPT1x5F7fBwCG3qHfnzeP/Vh/ZxCUA== + isolated-vm@^4.7.2: version "4.7.2" resolved "https://registry.yarnpkg.com/isolated-vm/-/isolated-vm-4.7.2.tgz#5670d5cce1d92004f9b825bec5b0b11fc7501b65" @@ -16254,7 +16303,7 @@ msgpackr-extract@^3.0.2: "@msgpackr-extract/msgpackr-extract-linux-x64" "3.0.2" "@msgpackr-extract/msgpackr-extract-win32-x64" "3.0.2" -msgpackr@^1.5.2: +msgpackr@1.10.1, msgpackr@^1.5.2: version "1.10.1" resolved "https://registry.yarnpkg.com/msgpackr/-/msgpackr-1.10.1.tgz#51953bb4ce4f3494f0c4af3f484f01cfbb306555" integrity sha512-r5VRLv9qouXuLiIBrLpl2d5ZvPt8svdQTl5/vMvE4nzDMyEX4sgW5yWhuBBj5UmgwOTWj8CIdSXn5sAfsHAWIQ== @@ -16448,25 +16497,13 @@ node-domexception@1.0.0: resolved "https://registry.yarnpkg.com/node-domexception/-/node-domexception-1.0.0.tgz#6888db46a1f71c0b76b3f7555016b63fe64766e5" integrity sha512-/jKZoMpw0F8GRwl4/eLROPA3cfcXtLApP0QzLmUT/HuPCZWyB7IY9ZrMeKw2O/nFIqPQB3PVM9aYm0F312AXDQ== -node-fetch@2.6.0: - version "2.6.0" - resolved "https://registry.yarnpkg.com/node-fetch/-/node-fetch-2.6.0.tgz#e633456386d4aa55863f676a7ab0daa8fdecb0fd" - integrity sha512-8dG4H5ujfvFiqDmVu9fQ5bOHUC15JMjMY/Zumv26oOvvVJjM67KF8koCWIabKQ1GJIa9r2mMZscBq/TbdOcmNA== - -node-fetch@2.6.7, node-fetch@^2.6.0, node-fetch@^2.6.1, node-fetch@^2.6.7: +node-fetch@2.6.0, node-fetch@2.6.7, node-fetch@^2.6.0, node-fetch@^2.6.1, node-fetch@^2.6.7, node-fetch@^2.6.9, node-fetch@^2.7.0: version "2.6.7" resolved "https://registry.yarnpkg.com/node-fetch/-/node-fetch-2.6.7.tgz#24de9fba827e3b4ae44dc8b20256a379160052ad" integrity sha512-ZjMPFEfVx5j+y2yF35Kzx5sF7kDzxuDj6ziH4FFbOp87zKDZNx8yExJIb05OGF4Nlt9IHFIMBkRl41VdvcNdbQ== dependencies: whatwg-url "^5.0.0" -node-fetch@^2.6.9, node-fetch@^2.7.0: - version "2.7.0" - resolved "https://registry.yarnpkg.com/node-fetch/-/node-fetch-2.7.0.tgz#d0f0fa6e3e2dc1d27efcd8ad99d550bda94d187d" - integrity sha512-c4FRfUm/dbcWZ7U+1Wq0AwCyFL+3nt2bEw05wfxSz+DWpWsitgmSgYmy2dQdWyKC1694ELPqMs/YzUSNozLt8A== - dependencies: - whatwg-url "^5.0.0" - node-forge@^1.2.1, node-forge@^1.3.1: version "1.3.1" resolved "https://registry.yarnpkg.com/node-forge/-/node-forge-1.3.1.tgz#be8da2af243b2417d5f646a770663a92b7e9ded3" @@ -17617,15 +17654,7 @@ passport-strategy@1.x.x, passport-strategy@^1.0.0: resolved "https://registry.yarnpkg.com/passport-strategy/-/passport-strategy-1.0.0.tgz#b5539aa8fc225a3d1ad179476ddf236b440f52e4" integrity sha512-CB97UUvDKJde2V0KDWWB3lyf6PC3FaZP7YxZ2G8OAtn9p4HI9j9JLP9qjOGZFvyl8uwNT8qM+hGnz/n16NI7oA== -passport@^0.4.0: - version "0.4.1" - resolved "https://registry.yarnpkg.com/passport/-/passport-0.4.1.tgz#941446a21cb92fc688d97a0861c38ce9f738f270" - integrity sha512-IxXgZZs8d7uFSt3eqNjM9NQ3g3uQCW5avD8mRNoXV99Yig50vjuaez6dQK2qC0kVWPRTujxY0dWgGfT09adjYg== - dependencies: - passport-strategy "1.x.x" - pause "0.0.1" - -passport@^0.6.0: +passport@0.6.0, passport@^0.4.0, passport@^0.6.0: version "0.6.0" resolved "https://registry.yarnpkg.com/passport/-/passport-0.6.0.tgz#e869579fab465b5c0b291e841e6cc95c005fac9d" integrity sha512-0fe+p3ZnrWRW74fe8+SvCyf4a3Pb2/h7gFkQ8yTJpAO50gDzlfjZUZTO1k5Eg9kUct22OxHLqDZoKUWRHOh9ug== @@ -18898,7 +18927,7 @@ pseudomap@^1.0.2: resolved "https://registry.yarnpkg.com/pseudomap/-/pseudomap-1.0.2.tgz#f052a28da70e618917ef0a8ac34c1ae5a68286b3" integrity sha512-b/YwNhb8lk1Zz2+bXXpS/LK9OisiZZ1SNsSLxN1x2OXVEhW2Ckr/7mWE5vrC1ZTiJlD9g19jWszTmJsB+oEpFQ== -psl@^1.1.28, psl@^1.1.33: +psl@^1.1.33: version "1.9.0" resolved "https://registry.yarnpkg.com/psl/-/psl-1.9.0.tgz#d0df2a137f00794565fcaf3b2c00cd09f8d5a5a7" integrity sha512-E/ZsdU4HLs/68gYzgGTkMicWTLPdAftJLfJFlLUAAKZGkStNU72sZjT66SnMDVOfOWY/YAoiD7Jxa9iHvngcag== @@ -19970,11 +19999,6 @@ sax@1.2.1: resolved "https://registry.yarnpkg.com/sax/-/sax-1.2.1.tgz#7b8e656190b228e81a66aea748480d828cd2d37a" integrity sha512-8I2a3LovHTOpm7NV5yOyO8IHqgVsfK4+UuySrXU8YXkSRX7k6hCV9b3HrkKCr3nMpgj+0bmocaJJWpvp1oc7ZA== -sax@>=0.1.1: - version "1.4.1" - resolved "https://registry.yarnpkg.com/sax/-/sax-1.4.1.tgz#44cc8988377f126304d3b3fc1010c733b929ef0f" - integrity sha512-+aWOz7yVScEGoKNd4PA10LZ8sk0A/z5+nXQG5giUO5rprX9jgYsTdov9qCchZiPIZezbZH+jRut8nPodFAX4Jg== - sax@>=0.6.0: version "1.2.4" resolved "https://registry.yarnpkg.com/sax/-/sax-1.2.4.tgz#2816234e2378bddc4e5354fab5caa895df7100d9" @@ -20047,33 +20071,13 @@ semver-diff@^3.1.1: dependencies: semver "^6.3.0" -"semver@2 || 3 || 4 || 5", semver@^5.5.0, semver@^5.6.0, semver@^5.7.1: - version "5.7.2" - resolved "https://registry.yarnpkg.com/semver/-/semver-5.7.2.tgz#48d55db737c3287cd4835e17fa13feace1c41ef8" - integrity sha512-cBznnQ9KjJqU67B52RMC65CMarK2600WFnbkcaiwWq3xy/5haFJlshgnpjovMVJ+Hff49d8GEn0b87C5pDQ10g== - -semver@7.5.3, semver@^7.0.0, semver@^7.1.1, semver@^7.3.2, semver@^7.3.4, semver@^7.3.5, semver@^7.3.7, semver@^7.3.8, semver@^7.5.3: +"semver@2 || 3 || 4 || 5", semver@7.5.3, semver@^5.5.0, semver@^5.6.0, semver@^5.7.1, semver@^6.0.0, semver@^6.1.1, semver@^6.1.2, semver@^6.2.0, semver@^6.3.0, semver@^6.3.1, semver@^7.0.0, semver@^7.1.1, semver@^7.3.2, semver@^7.3.4, semver@^7.3.5, semver@^7.3.7, semver@^7.3.8, semver@^7.5.3, semver@^7.5.4, semver@^7.6.0, semver@^7.6.3, semver@~2.3.1: version "7.5.3" resolved "https://registry.yarnpkg.com/semver/-/semver-7.5.3.tgz#161ce8c2c6b4b3bdca6caadc9fa3317a4c4fe88e" integrity sha512-QBlUtyVk/5EeHbi7X0fw6liDZc7BBmEaSYn01fMU1OUYbf6GPsbTtd8WmnqbI20SeycoHSeiybkE/q1Q+qlThQ== dependencies: lru-cache "^6.0.0" -semver@^6.0.0, semver@^6.1.1, semver@^6.1.2, semver@^6.2.0, semver@^6.3.0, semver@^6.3.1: - version "6.3.1" - resolved "https://registry.yarnpkg.com/semver/-/semver-6.3.1.tgz#556d2ef8689146e46dcea4bfdd095f3434dffcb4" - integrity sha512-BR7VvDCVHO+q2xBEWskxS6DJE1qRnb7DxzUrogb71CWoSficBxYsiAGd+Kl0mmq/MprG9yArRkyrQxTO6XjMzA== - -semver@^7.5.4, semver@^7.6.0, semver@^7.6.3: - version "7.6.3" - resolved "https://registry.yarnpkg.com/semver/-/semver-7.6.3.tgz#980f7b5550bc175fb4dc09403085627f9eb33143" - integrity sha512-oVekP1cKtI+CTDvHWYFUcMtsK/00wmAEfyqKfNdARm8u1wNVhSgaX7A8d4UuIlUI5e84iEwOhs7ZPYRmzU9U6A== - -semver@~2.3.1: - version "2.3.2" - resolved "https://registry.yarnpkg.com/semver/-/semver-2.3.2.tgz#b9848f25d6cf36333073ec9ef8856d42f1233e52" - integrity sha512-abLdIKCosKfpnmhS52NCTjO4RiLspDfsn37prjzGrp9im5DPJOgh82Os92vtwGh6XdQryKI/7SREZnV+aqiXrA== - seq-queue@^0.0.5: version "0.0.5" resolved "https://registry.yarnpkg.com/seq-queue/-/seq-queue-0.0.5.tgz#d56812e1c017a6e4e7c3e3a37a1da6d78dd3c93e" @@ -21636,7 +21640,7 @@ touch@^3.1.0: dependencies: nopt "~1.0.10" -"tough-cookie@^2.3.3 || ^3.0.1 || ^4.0.0", tough-cookie@^4.0.0, tough-cookie@^4.1.2: +tough-cookie@4.1.3, "tough-cookie@^2.3.3 || ^3.0.1 || ^4.0.0", tough-cookie@^4.0.0, tough-cookie@^4.1.2, tough-cookie@~2.5.0: version "4.1.3" resolved "https://registry.yarnpkg.com/tough-cookie/-/tough-cookie-4.1.3.tgz#97b9adb0728b42280aa3d814b6b999b2ff0318bf" integrity sha512-aX/y5pVRkfRnfmuX+OdbSdXvPe6ieKX/G2s7e98f4poJHnqH3281gDPm/metm6E/WRamfx7WC4HUqkWHfQHprw== @@ -21646,14 +21650,6 @@ touch@^3.1.0: universalify "^0.2.0" url-parse "^1.5.3" -tough-cookie@~2.5.0: - version "2.5.0" - resolved "https://registry.yarnpkg.com/tough-cookie/-/tough-cookie-2.5.0.tgz#cd9fb2a0aa1d5a12b473bd9fb96fa3dcff65ade2" - integrity sha512-nlLsUzgm1kfLXSXfRZMc1KLAugd4hqJHDTvc2hDIwS3mZAfMEuMbc03SujMF+GEcpaX/qboeycw6iO8JwVv2+g== - dependencies: - psl "^1.1.28" - punycode "^2.1.1" - tr46@^2.1.0: version "2.1.0" resolved "https://registry.yarnpkg.com/tr46/-/tr46-2.1.0.tgz#fa87aa81ca5d5941da8cbf1f9b749dc969a4e240" @@ -22182,6 +22178,14 @@ unpipe@1.0.0: resolved "https://registry.yarnpkg.com/unpipe/-/unpipe-1.0.0.tgz#b2bf4ee8514aae6165b4817829d21b2ef49904ec" integrity sha512-pjy2bYhSsufwWlKwPc+l3cN7+wuJlK6uz0YdJEOlQDbl6jo/YlPi4mb8agUkVC8BF7V8NuzeyPNqRksA3hztKQ== +unset-value@2.0.1: + version "2.0.1" + resolved "https://registry.yarnpkg.com/unset-value/-/unset-value-2.0.1.tgz#57bed0c22d26f28d69acde5df9a11b77c74d2df3" + integrity sha512-2hvrBfjUE00PkqN+q0XP6yRAOGrR06uSiUoIQGZkc7GxvQ9H7v8quUPNtZjMg4uux69i8HWpIjLPUKwCuRGyNg== + dependencies: + has-value "^2.0.2" + isobject "^4.0.0" + untildify@^4.0.0: version "4.0.0" resolved "https://registry.yarnpkg.com/untildify/-/untildify-4.0.0.tgz#2bc947b953652487e4600949fb091e3ae8cd919b" @@ -22961,33 +22965,10 @@ xml-parse-from-string@^1.0.0: resolved "https://registry.yarnpkg.com/xml-parse-from-string/-/xml-parse-from-string-1.0.1.tgz#a9029e929d3dbcded169f3c6e28238d95a5d5a28" integrity sha512-ErcKwJTF54uRzzNMXq2X5sMIy88zJvfN2DmdoQvy7PAFJ+tPRU6ydWuOKNMyfmOjdyBQTFREi60s0Y0SyI0G0g== -xml2js@0.1.x: - version "0.1.14" - resolved "https://registry.yarnpkg.com/xml2js/-/xml2js-0.1.14.tgz#5274e67f5a64c5f92974cd85139e0332adc6b90c" - integrity sha512-pbdws4PPPNc1HPluSUKamY4GWMk592K7qwcj6BExbVOhhubub8+pMda/ql68b6L3luZs/OGjGSB5goV7SnmgnA== - dependencies: - sax ">=0.1.1" - -xml2js@0.4.19: - version "0.4.19" - resolved "https://registry.yarnpkg.com/xml2js/-/xml2js-0.4.19.tgz#686c20f213209e94abf0d1bcf1efaa291c7827a7" - integrity sha512-esZnJZJOiJR9wWKMyuvSE1y6Dq5LCuJanqhxslH2bxM6duahNZ+HMpCLhBQGZkbX6xRf8x1Y2eJlgt2q3qo49Q== - dependencies: - sax ">=0.6.0" - xmlbuilder "~9.0.1" - -xml2js@0.5.0: - version "0.5.0" - resolved "https://registry.yarnpkg.com/xml2js/-/xml2js-0.5.0.tgz#d9440631fbb2ed800203fad106f2724f62c493b7" - integrity sha512-drPFnkQJik/O+uPKpqSgr22mpuFHqKdbS835iAQrUC73L2F5WkboIRd63ai/2Yg6I1jzifPFKH2NTK+cfglkIA== - dependencies: - sax ">=0.6.0" - xmlbuilder "~11.0.0" - -xml2js@^0.4.19, xml2js@^0.4.5: - version "0.4.23" - resolved "https://registry.yarnpkg.com/xml2js/-/xml2js-0.4.23.tgz#a0c69516752421eb2ac758ee4d4ccf58843eac66" - integrity sha512-ySPiMjM0+pLDftHgXY4By0uswI3SPKLDw/i3UXbnO8M/p28zqexCUoPmQFrYD+/1BzhGJSs2i1ERWKJAtiLrug== +xml2js@0.1.x, xml2js@0.4.19, xml2js@0.5.0, xml2js@0.6.2, xml2js@^0.4.19, xml2js@^0.4.5: + version "0.6.2" + resolved "https://registry.yarnpkg.com/xml2js/-/xml2js-0.6.2.tgz#dd0b630083aa09c161e25a4d0901e2b2a929b499" + integrity sha512-T4rieHaC1EXcES0Kxxj4JWgaUQHDk+qwHcYOCFHfiwKz7tOVPLq7Hjq9dM1WCMhylqMEfP7hMcOIChvotiZegA== dependencies: sax ">=0.6.0" xmlbuilder "~11.0.0" @@ -22997,11 +22978,6 @@ xmlbuilder@~11.0.0: resolved "https://registry.yarnpkg.com/xmlbuilder/-/xmlbuilder-11.0.1.tgz#be9bae1c8a046e76b31127726347d0ad7002beb3" integrity sha512-fDlsI/kFEx7gLvbecc0/ohLG50fugQp8ryHzMTuW9vSa1GJ0XYWKnhsUx7oie3G98+r56aTQIUB4kht42R3JvA== -xmlbuilder@~9.0.1: - version "9.0.7" - resolved "https://registry.yarnpkg.com/xmlbuilder/-/xmlbuilder-9.0.7.tgz#132ee63d2ec5565c557e20f4c22df9aca686b10d" - integrity sha512-7YXTQc3P2l9+0rjaUbLwMKRhtmwg1M1eDf6nag7urC7pIPYLD9W/jmzQ4ptRSUbodw5S0jfoGTflLemQibSpeQ== - xmlchars@^2.2.0: version "2.2.0" resolved "https://registry.yarnpkg.com/xmlchars/-/xmlchars-2.2.0.tgz#060fe1bcb7f9c76fe2a17db86a9bc3ab894210cb" From 30e31e12544076ae1813ef7f5aac82d6626f9120 Mon Sep 17 00:00:00 2001 From: Adria Navarro Date: Fri, 6 Sep 2024 13:22:50 +0200 Subject: [PATCH 49/85] Enrich view columns --- packages/server/src/sdk/app/views/index.ts | 5 ++++- packages/types/src/sdk/view.ts | 2 +- 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/packages/server/src/sdk/app/views/index.ts b/packages/server/src/sdk/app/views/index.ts index a0cffb2634..4e45fcda87 100644 --- a/packages/server/src/sdk/app/views/index.ts +++ b/packages/server/src/sdk/app/views/index.ts @@ -1,4 +1,5 @@ import { + FieldSchema, FieldType, RelationSchemaField, RenameColumn, @@ -176,7 +177,7 @@ export async function enrichSchema( } const relTable = tableCache[tableId] - const result: Record = {} + const result: Record = {} for (const relTableFieldName of Object.keys(relTable.schema)) { const relTableField = relTable.schema[relTableFieldName] @@ -191,6 +192,7 @@ export async function enrichSchema( const isVisible = !!viewFields[relTableFieldName]?.visible const isReadonly = !!viewFields[relTableFieldName]?.readonly result[relTableFieldName] = { + ...relTableField, visible: isVisible, readonly: isReadonly, } @@ -211,6 +213,7 @@ export async function enrichSchema( ...tableSchema[key], ...ui, order: anyViewOrder ? ui?.order ?? undefined : tableSchema[key].order, + columns: undefined, } if (schema[key].type === FieldType.LINK) { diff --git a/packages/types/src/sdk/view.ts b/packages/types/src/sdk/view.ts index b330db3950..96a6807b69 100644 --- a/packages/types/src/sdk/view.ts +++ b/packages/types/src/sdk/view.ts @@ -3,7 +3,7 @@ import { FieldSchema, RelationSchemaField, ViewV2 } from "../documents" export interface ViewV2Enriched extends ViewV2 { schema?: { [key: string]: FieldSchema & { - columns?: Record + columns?: Record } } } From 77be1cd8694d8299bdc882d9abff26a043bb445d Mon Sep 17 00:00:00 2001 From: Adria Navarro Date: Fri, 6 Sep 2024 17:01:13 +0200 Subject: [PATCH 50/85] Add metadata on related columns --- .../server/src/api/controllers/view/viewsV2.ts | 3 +++ packages/server/src/sdk/app/views/index.ts | 13 ++++++++----- packages/types/src/documents/app/view.ts | 3 +-- packages/types/src/sdk/view.ts | 14 ++++++++++++-- 4 files changed, 24 insertions(+), 9 deletions(-) diff --git a/packages/server/src/api/controllers/view/viewsV2.ts b/packages/server/src/api/controllers/view/viewsV2.ts index 90e80fe81d..40afb2e846 100644 --- a/packages/server/src/api/controllers/view/viewsV2.ts +++ b/packages/server/src/api/controllers/view/viewsV2.ts @@ -29,6 +29,9 @@ async function parseSchema(view: CreateViewRequest) { acc[key] = { visible: fieldSchema.visible, readonly: fieldSchema.readonly, + order: fieldSchema.order, + width: fieldSchema.width, + icon: fieldSchema.icon, } return acc }, {}) diff --git a/packages/server/src/sdk/app/views/index.ts b/packages/server/src/sdk/app/views/index.ts index 4e45fcda87..13d81d6802 100644 --- a/packages/server/src/sdk/app/views/index.ts +++ b/packages/server/src/sdk/app/views/index.ts @@ -1,5 +1,4 @@ import { - FieldSchema, FieldType, RelationSchemaField, RenameColumn, @@ -8,6 +7,7 @@ import { View, ViewFieldMetadata, ViewV2, + ViewV2ColumnEnriched, ViewV2Enriched, } from "@budibase/types" import { HTTPError } from "@budibase/backend-core" @@ -177,7 +177,7 @@ export async function enrichSchema( } const relTable = tableCache[tableId] - const result: Record = {} + const result: Record = {} for (const relTableFieldName of Object.keys(relTable.schema)) { const relTableField = relTable.schema[relTableFieldName] @@ -189,10 +189,13 @@ export async function enrichSchema( continue } - const isVisible = !!viewFields[relTableFieldName]?.visible - const isReadonly = !!viewFields[relTableFieldName]?.readonly + const viewFieldSchema = viewFields[relTableFieldName] + const isVisible = !!viewFieldSchema?.visible + const isReadonly = !!viewFieldSchema?.readonly result[relTableFieldName] = { - ...relTableField, + ...viewFieldSchema, + type: relTableField.type, + name: relTableField.name, visible: isVisible, readonly: isReadonly, } diff --git a/packages/types/src/documents/app/view.ts b/packages/types/src/documents/app/view.ts index b5fdcacefe..b847520526 100644 --- a/packages/types/src/documents/app/view.ts +++ b/packages/types/src/documents/app/view.ts @@ -38,8 +38,7 @@ export type ViewFieldMetadata = UIFieldMetadata & { columns?: Record } -export type RelationSchemaField = { - visible?: boolean +export type RelationSchemaField = UIFieldMetadata & { readonly?: boolean } diff --git a/packages/types/src/sdk/view.ts b/packages/types/src/sdk/view.ts index 96a6807b69..7480bf563f 100644 --- a/packages/types/src/sdk/view.ts +++ b/packages/types/src/sdk/view.ts @@ -1,9 +1,19 @@ -import { FieldSchema, RelationSchemaField, ViewV2 } from "../documents" +import { + FieldSchema, + FieldType, + RelationSchemaField, + ViewV2, +} from "../documents" export interface ViewV2Enriched extends ViewV2 { schema?: { [key: string]: FieldSchema & { - columns?: Record + columns?: Record } } } + +export interface ViewV2ColumnEnriched extends RelationSchemaField { + name: string + type: FieldType +} From 26db79d4218f7bc53fbaacfd014228f92c453887 Mon Sep 17 00:00:00 2001 From: Adria Navarro Date: Mon, 9 Sep 2024 16:24:20 +0200 Subject: [PATCH 51/85] Remove cache dependency --- .../controls/ColumnsSettingContent.svelte | 36 +++++++------------ 1 file changed, 13 insertions(+), 23 deletions(-) diff --git a/packages/frontend-core/src/components/grid/controls/ColumnsSettingContent.svelte b/packages/frontend-core/src/components/grid/controls/ColumnsSettingContent.svelte index 02ed0a504c..2c6a21d65b 100644 --- a/packages/frontend-core/src/components/grid/controls/ColumnsSettingContent.svelte +++ b/packages/frontend-core/src/components/grid/controls/ColumnsSettingContent.svelte @@ -12,7 +12,7 @@ export let columns export let fromRelationshipField - const { datasource, dispatch, cache, config } = getContext("grid") + const { datasource, dispatch, config } = getContext("grid") $: canSetRelationshipSchemas = $config.canSetRelationshipSchemas @@ -114,29 +114,19 @@ return { ...c, options } }) - let relationshipPanelColumns = [] - async function fetchRelationshipPanelColumns(relationshipField) { - relationshipPanelColumns = [] - if (!relationshipField) { - return + $: relationshipPanelColumns = Object.entries( + relationshipField?.columns || {} + ).map(([name, column]) => { + return { + name: name, + label: name, + schema: { + type: column.type, + visible: column.visible, + readonly: column.readonly, + }, } - - const table = await cache.actions.getTable(relationshipField.tableId) - relationshipPanelColumns = Object.entries( - relationshipField?.columns || {} - ).map(([name, column]) => { - return { - name: name, - label: name, - schema: { - type: table.schema[name].type, - visible: column.visible, - readonly: column.readonly, - }, - } - }) - } - $: fetchRelationshipPanelColumns(relationshipField) + }) async function toggleColumn(column, permission) { const visible = permission !== FieldPermissions.HIDDEN From e40a08ceca50f7f1189f388b6de2a22a3330ad42 Mon Sep 17 00:00:00 2001 From: Adria Navarro Date: Mon, 9 Sep 2024 16:27:54 +0200 Subject: [PATCH 52/85] Fix tests --- packages/server/src/sdk/app/views/tests/views.spec.ts | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/packages/server/src/sdk/app/views/tests/views.spec.ts b/packages/server/src/sdk/app/views/tests/views.spec.ts index 5a86702ab6..1d7360c5eb 100644 --- a/packages/server/src/sdk/app/views/tests/views.spec.ts +++ b/packages/server/src/sdk/app/views/tests/views.spec.ts @@ -355,10 +355,14 @@ describe("table sdk", () => { visible: true, columns: { title: { + name: "title", + type: "string", visible: true, readonly: true, }, age: { + name: "age", + type: "number", visible: false, readonly: false, }, From 1c5bab07aab725bdfb6ffeb9cec53d5165c9dc1c Mon Sep 17 00:00:00 2001 From: Adria Navarro Date: Mon, 9 Sep 2024 16:42:41 +0200 Subject: [PATCH 53/85] Fix tests --- .../src/api/routes/tests/viewV2.spec.ts | 90 +++++++++++++++---- 1 file changed, 72 insertions(+), 18 deletions(-) diff --git a/packages/server/src/api/routes/tests/viewV2.spec.ts b/packages/server/src/api/routes/tests/viewV2.spec.ts index 6d2d13e580..3ca28f31aa 100644 --- a/packages/server/src/api/routes/tests/viewV2.spec.ts +++ b/packages/server/src/api/routes/tests/viewV2.spec.ts @@ -1278,9 +1278,18 @@ describe.each([ schema: expect.objectContaining({ aux: expect.objectContaining({ columns: { - id: { visible: false, readonly: false }, - name: { visible: true, readonly: true }, - dob: { visible: true, readonly: true }, + id: expect.objectContaining({ + visible: false, + readonly: false, + }), + name: expect.objectContaining({ + visible: true, + readonly: true, + }), + dob: expect.objectContaining({ + visible: true, + readonly: true, + }), }, }), }), @@ -1323,16 +1332,34 @@ describe.each([ schema: expect.objectContaining({ aux: expect.objectContaining({ columns: { - id: { visible: false, readonly: false }, - name: { visible: true, readonly: true }, - dob: { visible: true, readonly: true }, + id: expect.objectContaining({ + visible: false, + readonly: false, + }), + name: expect.objectContaining({ + visible: true, + readonly: true, + }), + dob: expect.objectContaining({ + visible: true, + readonly: true, + }), }, }), aux2: expect.objectContaining({ columns: { - id: { visible: false, readonly: false }, - name: { visible: true, readonly: true }, - dob: { visible: true, readonly: true }, + id: expect.objectContaining({ + visible: false, + readonly: false, + }), + name: expect.objectContaining({ + visible: true, + readonly: true, + }), + dob: expect.objectContaining({ + visible: true, + readonly: true, + }), }, }), }), @@ -1375,16 +1402,34 @@ describe.each([ schema: expect.objectContaining({ aux: expect.objectContaining({ columns: { - id: { visible: false, readonly: false }, - fullName: { visible: true, readonly: true }, - age: { visible: false, readonly: false }, + id: expect.objectContaining({ + visible: false, + readonly: false, + }), + fullName: expect.objectContaining({ + visible: true, + readonly: true, + }), + age: expect.objectContaining({ + visible: false, + readonly: false, + }), }, }), aux2: expect.objectContaining({ columns: { - id: { visible: false, readonly: false }, - name: { visible: true, readonly: true }, - age: { visible: false, readonly: false }, + id: expect.objectContaining({ + visible: false, + readonly: false, + }), + name: expect.objectContaining({ + visible: true, + readonly: true, + }), + age: expect.objectContaining({ + visible: false, + readonly: false, + }), }, }), }), @@ -1427,9 +1472,18 @@ describe.each([ schema: expect.objectContaining({ aux: expect.objectContaining({ columns: { - id: { visible: false, readonly: false }, - name: { visible: true, readonly: true }, - dob: { visible: true, readonly: true }, + id: expect.objectContaining({ + visible: false, + readonly: false, + }), + name: expect.objectContaining({ + visible: true, + readonly: true, + }), + dob: expect.objectContaining({ + visible: true, + readonly: true, + }), }, }), }), From 1eb8c3409a5cbba0e8de6ead47370f5bfda1a03f Mon Sep 17 00:00:00 2001 From: Sam Rose Date: Mon, 9 Sep 2024 16:33:35 +0100 Subject: [PATCH 54/85] More progress toward a row save test passing. --- .../server/src/integrations/googlesheets.ts | 17 +- .../integrations/tests/googlesheets.spec.ts | 12 +- .../integrations/tests/utils/googlesheets.ts | 289 ++++++++++++------ 3 files changed, 219 insertions(+), 99 deletions(-) diff --git a/packages/server/src/integrations/googlesheets.ts b/packages/server/src/integrations/googlesheets.ts index d8d775c629..4fc47500eb 100644 --- a/packages/server/src/integrations/googlesheets.ts +++ b/packages/server/src/integrations/googlesheets.ts @@ -395,7 +395,12 @@ export class GoogleSheetsIntegration implements DatasourcePlus { sheet, }) case Operation.CREATE_TABLE: - return this.createTable(json?.table?.name) + if (json.table === undefined) { + throw new Error( + "attempted to create a table without specifying the table to create" + ) + } + return this.createTable(json.table) case Operation.UPDATE_TABLE: return this.updateTable(json.table!) case Operation.DELETE_TABLE: @@ -422,13 +427,13 @@ export class GoogleSheetsIntegration implements DatasourcePlus { return rowObject } - private async createTable(name?: string) { - if (!name) { - throw new Error("Must provide name for new sheet.") - } + private async createTable(table: Table) { try { await this.connect() - await this.client.addSheet({ title: name, headerValues: [name] }) + await this.client.addSheet({ + title: table.name, + headerValues: Object.keys(table.schema), + }) } catch (err) { console.error("Error creating new table in google sheets", err) throw err diff --git a/packages/server/src/integrations/tests/googlesheets.spec.ts b/packages/server/src/integrations/tests/googlesheets.spec.ts index 25a3706e10..e31d3e4330 100644 --- a/packages/server/src/integrations/tests/googlesheets.spec.ts +++ b/packages/server/src/integrations/tests/googlesheets.spec.ts @@ -49,12 +49,11 @@ describe("Google Sheets Integration", () => { beforeEach(async () => { nock.cleanAll() mock = GoogleSheetsMock.forDatasource(datasource) - mock.init() }) describe("create", () => { it("creates a new table", async () => { - const table = await config.api.table.save({ + await config.api.table.save({ name: "Test Table", type: "table", sourceId: datasource._id!, @@ -77,11 +76,10 @@ describe("Google Sheets Integration", () => { }, }) - const cell = mock.getCell(table.name, "A1") - if (!cell) { - throw new Error("Cell not found") - } - expect(cell.userEnteredValue.stringValue).toEqual(table.name) + expect(mock.cell("A1")).toEqual("name") + expect(mock.cell("B1")).toEqual("description") + expect(mock.cell("A2")).toEqual(null) + expect(mock.cell("B2")).toEqual(null) }) }) diff --git a/packages/server/src/integrations/tests/utils/googlesheets.ts b/packages/server/src/integrations/tests/utils/googlesheets.ts index 82a274405b..70481e0e0d 100644 --- a/packages/server/src/integrations/tests/utils/googlesheets.ts +++ b/packages/server/src/integrations/tests/utils/googlesheets.ts @@ -3,7 +3,7 @@ import nock from "nock" import { GoogleSheetsConfig } from "../../googlesheets" // https://protobuf.dev/reference/protobuf/google.protobuf/#value -type Value = string | number | boolean +type Value = string | number | boolean | null // https://developers.google.com/sheets/api/reference/rest/v4/Dimension type Dimension = "ROWS" | "COLUMNS" @@ -195,6 +195,9 @@ export class GoogleSheetsMock { spreadsheetId: config.spreadsheetId, sheets: [], } + + this.mockAuth() + this.mockAPI() } private route( @@ -237,6 +240,7 @@ export class GoogleSheetsMock { client_secret: "your-client-secret", }) .persist() + nock("https://oauth2.googleapis.com/") .post("/token", { client_id: "test", @@ -253,9 +257,7 @@ export class GoogleSheetsMock { .persist() } - init() { - this.mockAuth() - + private mockAPI() { this.get(`/v4/spreadsheets/${this.config.spreadsheetId}/`, () => this.handleGetSpreadsheet() ) @@ -280,7 +282,7 @@ export class GoogleSheetsMock { if (!range) { throw new Error("No range provided") } - return this.handleGetValues(decodeURIComponent(range)) + return this.getValueRange(decodeURIComponent(range)) } ) @@ -301,11 +303,17 @@ export class GoogleSheetsMock { params.includeValuesInResponse = false } - const range = url.pathname.split("/").pop() + let range = url.pathname.split("/").pop() if (!range) { throw new Error("No range provided") } + if (range.endsWith(":append")) { + range = range.slice(0, -7) + } + + range = decodeURIComponent(range) + return this.handleValueAppend({ range, params, @@ -315,29 +323,44 @@ export class GoogleSheetsMock { ) } - private handleValueAppend(request: AppendRequest): AppendResponse {} + private handleValueAppend(request: AppendRequest): AppendResponse { + const { range, params, body } = request + const { sheet, bottomRight } = this.parseA1Notation(range) - private handleGetValues(range: string): ValueRange { - const { sheet, topLeft, bottomRight } = this.parseA1Notation(range) - const valueRange: ValueRange = { - range, - majorDimension: "ROWS", - values: [], - } + const newRows = body.values.map(v => this.valuesToRowData(v)) + const toDelete = + params.insertDataOption === "INSERT_ROWS" ? newRows.length : 0 + sheet.data[0].rowData.splice(bottomRight.row + 1, toDelete, ...newRows) + sheet.data[0].rowMetadata.splice(bottomRight.row + 1, toDelete, { + hiddenByUser: false, + hiddenByFilter: false, + pixelSize: 100, + }) - for (let row = topLeft.row; row <= bottomRight.row; row++) { - const values: Value[] = [] - for (let col = topLeft.column; col <= bottomRight.column; col++) { - const cell = this.getCellNumericIndexes(sheet, row, col) - if (!cell) { - throw new Error("Cell not found") - } - values.push(this.unwrapValue(cell.userEnteredValue)) + const updatedRange = this.createA1FromRanges( + sheet, + { + row: bottomRight.row + 1, + column: 0, + }, + { + row: bottomRight.row + newRows.length, + column: 0, } - valueRange.values.push(values) - } + ) - return valueRange + return { + spreadsheetId: this.spreadsheet.spreadsheetId, + tableRange: range, + updates: { + spreadsheetId: this.spreadsheet.spreadsheetId, + updatedRange, + updatedRows: body.values.length, + updatedColumns: body.values[0].length, + updatedCells: body.values.length * body.values[0].length, + updatedData: body, + }, + } } private handleBatchUpdate( @@ -387,29 +410,9 @@ export class GoogleSheetsMock { } private handleValueUpdate(valueRange: ValueRange): UpdateValuesResponse { - if (valueRange.majorDimension !== "ROWS") { - throw new Error("Only row-major updates are supported") - } - - const { sheet, topLeft, bottomRight } = this.parseA1Notation( - valueRange.range - ) - - for (let row = topLeft.row; row <= bottomRight.row; row++) { - for ( - let column = topLeft.column; - column <= bottomRight.column; - column++ - ) { - const cell = this.getCellNumericIndexes(sheet, row, column) - if (!cell) { - continue - } - const value = - valueRange.values[row - topLeft.row][column - topLeft.column] - cell.userEnteredValue = this.createValue(value) - } - } + this.iterateCells(valueRange, (cell, value) => { + cell.userEnteredValue = this.createValue(value) + }) const response: UpdateValuesResponse = { spreadsheetId: this.spreadsheet.spreadsheetId, @@ -422,6 +425,60 @@ export class GoogleSheetsMock { return response } + private iterateCells( + valueRange: ValueRange, + cb: (cell: CellData, value: Value) => void + ) { + if (valueRange.majorDimension !== "ROWS") { + throw new Error("Only row-major updates are supported") + } + + const { sheet, topLeft, bottomRight } = this.parseA1Notation( + valueRange.range + ) + for (let row = topLeft.row; row <= bottomRight.row; row++) { + for (let col = topLeft.column; col <= bottomRight.column; col++) { + const cell = this.getCellNumericIndexes(sheet, row, col) + if (!cell) { + throw new Error("Cell not found") + } + const value = valueRange.values[row - topLeft.row][col - topLeft.column] + cb(cell, value) + } + } + } + + private getValueRange(range: string): ValueRange { + const { sheet, topLeft, bottomRight } = this.parseA1Notation(range) + const valueRange: ValueRange = { + range, + majorDimension: "ROWS", + values: [], + } + + for (let row = topLeft.row; row <= bottomRight.row; row++) { + const values: Value[] = [] + for (let col = topLeft.column; col <= bottomRight.column; col++) { + const cell = this.getCellNumericIndexes(sheet, row, col) + if (!cell) { + throw new Error("Cell not found") + } + values.push(this.unwrapValue(cell.userEnteredValue)) + } + valueRange.values.push(values) + } + + return valueRange + } + + private valuesToRowData(values: Value[]): RowData { + return { + values: values.map(v => { + return { userEnteredValue: this.createValue(v) } + }), + } + } + private unwrapValue(from: ExtendedValue): Value { if (from.stringValue !== undefined) { return from.stringValue @@ -432,12 +489,14 @@ export class GoogleSheetsMock { } else if (from.formulaValue !== undefined) { return from.formulaValue } else { - throw new Error("Unsupported value type") + return null } } private createValue(from: Value): ExtendedValue { - if (typeof from === "string") { + if (from == null) { + return {} + } else if (typeof from === "string") { return { stringValue: from, } @@ -459,15 +518,9 @@ export class GoogleSheetsMock { for (let row = 0; row < numRows; row++) { const cells: CellData[] = [] for (let col = 0; col < numCols; col++) { - cells.push({ - userEnteredValue: { - stringValue: "", - }, - }) + cells.push({ userEnteredValue: this.createValue(null) }) } - rowData.push({ - values: cells, - }) + rowData.push({ values: cells }) } const rowMetadata: DimensionProperties[] = [] for (let row = 0; row < numRows; row++) { @@ -495,16 +548,20 @@ export class GoogleSheetsMock { } } - getCell(sheetName: string, ref: string): CellData | undefined { - const sheet = this.getSheetByName(sheetName) - if (!sheet) { + private cellData(cell: string): CellData | undefined { + const { + sheet, + topLeft: { row, column }, + } = this.parseA1Notation(cell) + return this.getCellNumericIndexes(sheet, row, column) + } + + cell(cell: string): Value | undefined { + const cellData = this.cellData(cell) + if (!cellData) { return undefined } - const { row, column } = this.parseCell(ref) - if (row === "ALL" || column === "ALL") { - throw new Error("Only single cell references are supported") - } - return this.getCellNumericIndexes(sheet, row, column) + return this.unwrapValue(cellData.userEnteredValue) } private getCellNumericIndexes( @@ -525,36 +582,83 @@ export class GoogleSheetsMock { } // https://developers.google.com/sheets/api/guides/concepts#cell + // + // Examples from + // https://code.luasoftware.com/tutorials/google-sheets-api/google-sheets-api-range-parameter-a1-notation + // + // "Sheet1!A1" -> First cell on Row 1 Col 1 + // "Sheet1!A1:C1" -> Col 1-3 (A, B, C) on Row 1 = A1, B1, C1 + // "A1" -> First visible sheet (if sheet name is ommitted) + // "'My Sheet'!A1" -> If sheet name which contain space or start with a bracket. + // "Sheet1" -> All cells in Sheet1. + // "Sheet1!A:A" -> All cells on Col 1. + // "Sheet1!A:B" -> All cells on Col 1 and 2. + // "Sheet1!1:1" -> All cells on Row 1. + // "Sheet1!1:2" -> All cells on Row 1 and 2. + // + // How that translates to our code below, omitting the `sheet` property: + // + // "Sheet1!A1" -> { topLeft: { row: 0, column: 0 }, bottomRight: { row: 0, column: 0 } } + // "Sheet1!A1:C1" -> { topLeft: { row: 0, column: 0 }, bottomRight: { row: 0, column: 2 } } + // "A1" -> { topLeft: { row: 0, column: 0 }, bottomRight: { row: 0, column: 0 } } + // "Sheet1" -> { topLeft: { row: 0, column: 0 }, bottomRight: { row: 100, column: 25 } } + // -> This is because we default to having a 100x26 grid. + // "Sheet1!A:A" -> { topLeft: { row: 0, column: 0 }, bottomRight: { row: 99, column: 0 } } + // "Sheet1!A:B" -> { topLeft: { row: 0, column: 0 }, bottomRight: { row: 99, column: 1 } } + // "Sheet1!1:1" -> { topLeft: { row: 0, column: 0 }, bottomRight: { row: 0, column: 25 } } + // "Sheet1!1:2" -> { topLeft: { row: 0, column: 0 }, bottomRight: { row: 1, column: 25 } } private parseA1Notation(range: string): { sheet: Sheet topLeft: Range bottomRight: Range } { - let [sheetName, rest] = range.split("!") + let sheet: Sheet + let rest: string + if (!range.includes("!")) { + sheet = this.spreadsheet.sheets[0] + rest = range + } else { + let sheetName = range.split("!")[0] + if (sheetName.startsWith("'") && sheetName.endsWith("'")) { + sheetName = sheetName.slice(1, -1) + } + const foundSheet = this.getSheetByName(sheetName) + if (!foundSheet) { + throw new Error(`Sheet ${sheetName} not found`) + } + sheet = foundSheet + rest = range.split("!")[1] + } + const [topLeft, bottomRight] = rest.split(":") - if (sheetName.startsWith("'") && sheetName.endsWith("'")) { - sheetName = sheetName.slice(1, -1) + const parsedTopLeft = topLeft ? this.parseCell(topLeft) : undefined + let parsedBottomRight = bottomRight + ? this.parseCell(bottomRight) + : undefined + + if (!parsedTopLeft && !parsedBottomRight) { + throw new Error("No range provided") } - const sheet = this.getSheetByName(sheetName) - if (!sheet) { - throw new Error(`Sheet ${sheetName} not found`) + if (!parsedTopLeft) { + throw new Error("No top left cell provided") } - const parsedTopLeft = this.parseCell(topLeft) - const parsedBottomRight = this.parseCell(bottomRight) + if (!parsedBottomRight) { + parsedBottomRight = parsedTopLeft + } - if (parsedTopLeft.row === "ALL") { + if (parsedTopLeft && parsedTopLeft.row === undefined) { parsedTopLeft.row = 0 } - if (parsedBottomRight.row === "ALL") { - parsedBottomRight.row = sheet.properties.gridProperties.rowCount - 1 - } - if (parsedTopLeft.column === "ALL") { + if (parsedTopLeft && parsedTopLeft.column === undefined) { parsedTopLeft.column = 0 } - if (parsedBottomRight.column === "ALL") { + if (parsedBottomRight && parsedBottomRight.row === undefined) { + parsedBottomRight.row = sheet.properties.gridProperties.rowCount - 1 + } + if (parsedBottomRight && parsedBottomRight.column === undefined) { parsedBottomRight.column = sheet.properties.gridProperties.columnCount - 1 } @@ -565,22 +669,31 @@ export class GoogleSheetsMock { } } + private createA1FromRanges(sheet: Sheet, topLeft: Range, bottomRight: Range) { + let title = sheet.properties.title + if (title.includes(" ")) { + title = `'${title}'` + } + const topLeftLetter = this.numberToLetter(topLeft.column) + const bottomRightLetter = this.numberToLetter(bottomRight.column) + const topLeftRow = topLeft.row + 1 + const bottomRightRow = bottomRight.row + 1 + return `${title}!${topLeftLetter}${topLeftRow}:${bottomRightLetter}${bottomRightRow}` + } + /** * Parses a cell reference into a row and column. * @param cell a string of the form A1, B2, etc. * @returns */ - private parseCell(cell: string): { - row: number | "ALL" - column: number | "ALL" - } { + private parseCell(cell: string): Partial { const firstChar = cell.slice(0, 1) if (this.isInteger(firstChar)) { - return { row: parseInt(cell) - 1, column: "ALL" } + return { row: parseInt(cell) - 1 } } const column = this.letterToNumber(firstChar) if (cell.length === 1) { - return { row: "ALL", column } + return { column } } const number = cell.slice(1) return { row: parseInt(number) - 1, column } @@ -594,6 +707,10 @@ export class GoogleSheetsMock { return letter.charCodeAt(0) - 65 } + private numberToLetter(number: number): string { + return String.fromCharCode(number + 65) + } + private getSheetByName(name: string): Sheet | undefined { return this.spreadsheet.sheets.find( sheet => sheet.properties.title === name From 9e9f14d1b744148106318d4a68967bfcd14f9414 Mon Sep 17 00:00:00 2001 From: Sam Rose Date: Mon, 9 Sep 2024 16:45:15 +0100 Subject: [PATCH 55/85] More comments. --- .../src/integrations/tests/utils/googlesheets.ts | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/packages/server/src/integrations/tests/utils/googlesheets.ts b/packages/server/src/integrations/tests/utils/googlesheets.ts index 70481e0e0d..a690a105fb 100644 --- a/packages/server/src/integrations/tests/utils/googlesheets.ts +++ b/packages/server/src/integrations/tests/utils/googlesheets.ts @@ -1,3 +1,12 @@ +// In this file is a mock implementation of the Google Sheets API. It is used +// to test the Google Sheets integration, and it keeps track of a single +// spreadsheet with many sheets. It aims to be a faithful recreation of the +// Google Sheets API, but it is not a perfect recreation. Some fields are +// missing if they aren't relevant to our use of the API. It's possible that +// this will cause problems for future feature development, but the original +// development of these tests involved hitting Google's APIs directly and +// examining the responses. If we couldn't find a good example of something in +// use, it wasn't included. import { Datasource } from "@budibase/types" import nock from "nock" import { GoogleSheetsConfig } from "../../googlesheets" @@ -13,6 +22,7 @@ interface Range { column: number } +// https://developers.google.com/sheets/api/reference/rest/v4/spreadsheets/sheets#DimensionProperties interface DimensionProperties { hiddenByFilter: boolean hiddenByUser: boolean From b4a4f81308fd6c6cccb91abaaee45a58828a5d48 Mon Sep 17 00:00:00 2001 From: Martin McKeaveney Date: Mon, 9 Sep 2024 16:52:56 +0100 Subject: [PATCH 56/85] linting --- packages/account-portal | 2 +- .../backend-core/src/events/publishers/ai.ts | 21 +++++++++++++++++++ .../src/events/publishers/index.ts | 1 + .../portal/settings/ai/ConfigModal.svelte | 6 +++++- packages/pro | 2 +- packages/types/src/documents/global/quotas.ts | 2 ++ packages/types/src/sdk/events/ai.ts | 5 +++++ packages/types/src/sdk/events/event.ts | 8 +++++++ packages/types/src/sdk/events/index.ts | 1 + .../src/api/controllers/global/configs.ts | 11 ++++++++++ 10 files changed, 56 insertions(+), 3 deletions(-) create mode 100644 packages/backend-core/src/events/publishers/ai.ts create mode 100644 packages/types/src/sdk/events/ai.ts diff --git a/packages/account-portal b/packages/account-portal index c24374879d..ce60bf9231 160000 --- a/packages/account-portal +++ b/packages/account-portal @@ -1 +1 @@ -Subproject commit c24374879d2b61516fabc24d7404e7da235be05e +Subproject commit ce60bf923115855a9cd0bb991297cccd308eae13 diff --git a/packages/backend-core/src/events/publishers/ai.ts b/packages/backend-core/src/events/publishers/ai.ts new file mode 100644 index 0000000000..168ac9f796 --- /dev/null +++ b/packages/backend-core/src/events/publishers/ai.ts @@ -0,0 +1,21 @@ +import { publishEvent } from "../events" +import { + Event, + AIConfigCreatedEvent, + AIConfigUpdatedEvent, +} from "@budibase/types" + +async function AIConfigCreated(timestamp?: string | number) { + const properties: AIConfigCreatedEvent = {} + await publishEvent(Event.AI_CONFIG_CREATED, properties, timestamp) +} + +async function AIConfigUpdated() { + const properties: AIConfigUpdatedEvent = {} + await publishEvent(Event.AI_CONFIG_UPDATED, properties) +} + +export default { + AIConfigCreated, + AIConfigUpdated, +} diff --git a/packages/backend-core/src/events/publishers/index.ts b/packages/backend-core/src/events/publishers/index.ts index 87a34bf3f1..9c92b80499 100644 --- a/packages/backend-core/src/events/publishers/index.ts +++ b/packages/backend-core/src/events/publishers/index.ts @@ -4,6 +4,7 @@ export { default as auth } from "./auth" export { default as automation } from "./automation" export { default as datasource } from "./datasource" export { default as email } from "./email" +export { default as ai } from "./ai" export { default as license } from "./license" export { default as layout } from "./layout" export { default as org } from "./org" diff --git a/packages/builder/src/pages/builder/portal/settings/ai/ConfigModal.svelte b/packages/builder/src/pages/builder/portal/settings/ai/ConfigModal.svelte index 2f7486d087..d480689028 100644 --- a/packages/builder/src/pages/builder/portal/settings/ai/ConfigModal.svelte +++ b/packages/builder/src/pages/builder/portal/settings/ai/ConfigModal.svelte @@ -54,7 +54,11 @@
- +
diff --git a/packages/pro b/packages/pro index a4d1d15d9c..62586bc47b 160000 --- a/packages/pro +++ b/packages/pro @@ -1 +1 @@ -Subproject commit a4d1d15d9ce6ac3deedb2e42625c90ba32756758 +Subproject commit 62586bc47bf2af507356b0e1e64df8b0ae9fad43 diff --git a/packages/types/src/documents/global/quotas.ts b/packages/types/src/documents/global/quotas.ts index 4eb1168f7d..4726dabf0d 100644 --- a/packages/types/src/documents/global/quotas.ts +++ b/packages/types/src/documents/global/quotas.ts @@ -35,6 +35,7 @@ export interface StaticUsage { [StaticQuotaName.CREATORS]: number [StaticQuotaName.USER_GROUPS]: number [StaticQuotaName.ROWS]: number + [StaticQuotaName.AI_CUSTOM_CONFIGS]: number triggers: { [key in StaticQuotaName]?: QuotaTriggers } @@ -44,6 +45,7 @@ export interface MonthlyUsage { [MonthlyQuotaName.QUERIES]: number [MonthlyQuotaName.AUTOMATIONS]: number [MonthlyQuotaName.DAY_PASSES]: number + [MonthlyQuotaName.BUDIBASE_AI_CREDITS]: number triggers: { [key in MonthlyQuotaName]?: QuotaTriggers } diff --git a/packages/types/src/sdk/events/ai.ts b/packages/types/src/sdk/events/ai.ts new file mode 100644 index 0000000000..740c59d5f9 --- /dev/null +++ b/packages/types/src/sdk/events/ai.ts @@ -0,0 +1,5 @@ +import { BaseEvent } from "./event" + +export interface AIConfigCreatedEvent extends BaseEvent {} + +export interface AIConfigUpdatedEvent extends BaseEvent {} diff --git a/packages/types/src/sdk/events/event.ts b/packages/types/src/sdk/events/event.ts index 05f4fe4be9..242b182dec 100644 --- a/packages/types/src/sdk/events/event.ts +++ b/packages/types/src/sdk/events/event.ts @@ -33,6 +33,10 @@ export enum Event { EMAIL_SMTP_CREATED = "email:smtp:created", EMAIL_SMTP_UPDATED = "email:smtp:updated", + // AI + AI_CONFIG_CREATED = "ai:config:created", + AI_CONFIG_UPDATED = "ai:config:updated", + // AUTH AUTH_SSO_CREATED = "auth:sso:created", AUTH_SSO_UPDATED = "auth:sso:updated", @@ -243,6 +247,10 @@ export const AuditedEventFriendlyName: Record = { [Event.EMAIL_SMTP_CREATED]: `Email configuration created`, [Event.EMAIL_SMTP_UPDATED]: `Email configuration updated`, + // AI + [Event.AI_CONFIG_CREATED]: `AI configuration created`, + [Event.AI_CONFIG_UPDATED]: `AI configuration updated`, + // AUTH [Event.AUTH_SSO_CREATED]: `SSO configuration created`, [Event.AUTH_SSO_UPDATED]: `SSO configuration updated`, diff --git a/packages/types/src/sdk/events/index.ts b/packages/types/src/sdk/events/index.ts index 745f84d2a3..043e62faa4 100644 --- a/packages/types/src/sdk/events/index.ts +++ b/packages/types/src/sdk/events/index.ts @@ -2,6 +2,7 @@ export * from "./app" export * from "./auth" export * from "./automation" export * from "./email" +export * from "./ai" export * from "./datasource" export * from "./event" export * from "./layout" diff --git a/packages/worker/src/api/controllers/global/configs.ts b/packages/worker/src/api/controllers/global/configs.ts index d3eef71240..b8feb6158f 100644 --- a/packages/worker/src/api/controllers/global/configs.ts +++ b/packages/worker/src/api/controllers/global/configs.ts @@ -31,6 +31,7 @@ import { OIDCLogosConfig, AIConfig, PASSWORD_REPLACEMENT, + isAIConfig, } from "@budibase/types" import * as pro from "@budibase/pro" @@ -40,6 +41,11 @@ const getEventFns = async (config: Config, existing?: Config) => { if (!existing) { if (isSMTPConfig(config)) { fns.push(events.email.SMTPCreated) + } else if (isAIConfig(config)) { + fns.push(() => events.ai.AIConfigCreated) + fns.push(() => + pro.quotas.updateCustomAIConfigCount(Object.keys(config.config).length) + ) } else if (isGoogleConfig(config)) { fns.push(() => events.auth.SSOCreated(ConfigType.GOOGLE)) if (config.config.activated) { @@ -76,6 +82,11 @@ const getEventFns = async (config: Config, existing?: Config) => { } else { if (isSMTPConfig(config)) { fns.push(events.email.SMTPUpdated) + } else if (isAIConfig(config)) { + fns.push(() => events.ai.AIConfigUpdated) + fns.push(() => + pro.quotas.updateCustomAIConfigCount(Object.keys(config.config).length) + ) } else if (isGoogleConfig(config)) { fns.push(() => events.auth.SSOUpdated(ConfigType.GOOGLE)) if (!existing.config.activated && config.config.activated) { From 1f405da3c306fd11444ac4d27ca42829282e53ea Mon Sep 17 00:00:00 2001 From: Sam Rose Date: Mon, 9 Sep 2024 17:51:32 +0100 Subject: [PATCH 57/85] Add some more tests. --- .../src/api/controllers/table/external.ts | 1 - .../server/src/integrations/googlesheets.ts | 25 ++----- .../integrations/tests/googlesheets.spec.ts | 66 +++++++++++++++++- .../integrations/tests/utils/googlesheets.ts | 69 +++++++++++++++---- 4 files changed, 126 insertions(+), 35 deletions(-) diff --git a/packages/server/src/api/controllers/table/external.ts b/packages/server/src/api/controllers/table/external.ts index fbdb5c0b3a..c3356919c8 100644 --- a/packages/server/src/api/controllers/table/external.ts +++ b/packages/server/src/api/controllers/table/external.ts @@ -53,7 +53,6 @@ export async function save( builderSocket?.emitDatasourceUpdate(ctx, datasource) return table } catch (err: any) { - throw err if (err instanceof Error) { ctx.throw(400, err.message) } else { diff --git a/packages/server/src/integrations/googlesheets.ts b/packages/server/src/integrations/googlesheets.ts index d6d19e2fd9..960d01ab52 100644 --- a/packages/server/src/integrations/googlesheets.ts +++ b/packages/server/src/integrations/googlesheets.ts @@ -378,6 +378,8 @@ export class GoogleSheetsIntegration implements DatasourcePlus { return this.create({ sheet, row: json.body as Row }) case Operation.BULK_CREATE: return this.createBulk({ sheet, rows: json.body as Row[] }) + case Operation.BULK_UPSERT: + return this.createBulk({ sheet, rows: json.body as Row[] }) case Operation.READ: return this.read({ ...json, sheet }) case Operation.UPDATE: @@ -557,32 +559,15 @@ export class GoogleSheetsIntegration implements DatasourcePlus { } else { rows = await sheet.getRows() } - // this is a special case - need to handle the _id, it doesn't exist - // we cannot edit the returned structure from google, it does not have - // setter functions and is immutable, easier to update the filters - // to look for the _rowNumber property rather than rowNumber - if (query.filters?.equal) { - const idFilterKeys = Object.keys(query.filters.equal).filter(filter => - filter.includes(GOOGLE_SHEETS_PRIMARY_KEY) - ) - for (let idFilterKey of idFilterKeys) { - const id = query.filters.equal[idFilterKey] - delete query.filters.equal[idFilterKey] - query.filters.equal[`_${GOOGLE_SHEETS_PRIMARY_KEY}`] = id - } - } if (hasFilters && query.paginate) { rows = rows.slice(offset, offset + limit) } const headerValues = sheet.headerValues - let response = [] - for (let row of rows) { - response.push( - this.buildRowObject(headerValues, row.toObject(), row.rowNumber) - ) - } + let response = rows.map(row => + this.buildRowObject(headerValues, row.toObject(), row.rowNumber) + ) response = dataFilters.runQuery(response, query.filters || {}) if (query.sort) { diff --git a/packages/server/src/integrations/tests/googlesheets.spec.ts b/packages/server/src/integrations/tests/googlesheets.spec.ts index e31d3e4330..d3044d17eb 100644 --- a/packages/server/src/integrations/tests/googlesheets.spec.ts +++ b/packages/server/src/integrations/tests/googlesheets.spec.ts @@ -104,11 +104,73 @@ describe("Google Sheets Integration", () => { }) }) - it.only("should be able to add a new row", async () => { - await config.api.row.save(table._id!, { + it("should be able to add a new row", async () => { + const row = await config.api.row.save(table._id!, { name: "Test Contact", description: "original description", }) + + expect(row.name).toEqual("Test Contact") + expect(row.description).toEqual("original description") + + expect(mock.cell("A2")).toEqual("Test Contact") + expect(mock.cell("B2")).toEqual("original description") + + const row2 = await config.api.row.save(table._id!, { + name: "Test Contact 2", + description: "original description 2", + }) + + expect(row2.name).toEqual("Test Contact 2") + expect(row2.description).toEqual("original description 2") + + // Notable that adding a new row adds it at the top, not the bottom. Not + // entirely sure if this is the intended behaviour or an incorrect + // implementation of the GoogleSheetsMock. + expect(mock.cell("A2")).toEqual("Test Contact 2") + expect(mock.cell("B2")).toEqual("original description 2") + + expect(mock.cell("A3")).toEqual("Test Contact") + expect(mock.cell("B3")).toEqual("original description") + }) + + it("should be able to add multiple rows", async () => { + await config.api.row.bulkImport(table._id!, { + rows: [ + { + name: "Test Contact 1", + description: "original description 1", + }, + { + name: "Test Contact 2", + description: "original description 2", + }, + ], + }) + + expect(mock.cell("A2")).toEqual("Test Contact 1") + expect(mock.cell("B2")).toEqual("original description 1") + expect(mock.cell("A3")).toEqual("Test Contact 2") + expect(mock.cell("B3")).toEqual("original description 2") + }) + + it("should be able to update a row", async () => { + const row = await config.api.row.save(table._id!, { + name: "Test Contact", + description: "original description", + }) + + expect(mock.cell("A2")).toEqual("Test Contact") + expect(mock.cell("B2")).toEqual("original description") + + await config.api.row.save(table._id!, { + ...row, + name: "Test Contact Updated", + description: "original description updated", + }) + + expect(mock.cell("A2")).toEqual("Test Contact Updated") + expect(mock.cell("B2")).toEqual("original description updated") }) }) }) diff --git a/packages/server/src/integrations/tests/utils/googlesheets.ts b/packages/server/src/integrations/tests/utils/googlesheets.ts index a690a105fb..4851104c8b 100644 --- a/packages/server/src/integrations/tests/utils/googlesheets.ts +++ b/packages/server/src/integrations/tests/utils/googlesheets.ts @@ -175,6 +175,20 @@ interface AppendParams { responseDateTimeRenderOption?: DateTimeRenderOption } +// https://developers.google.com/sheets/api/reference/rest/v4/spreadsheets.values/batchGet#query-parameters +interface BatchGetParams { + ranges: string[] + majorDimension?: Dimension + valueRenderOption?: ValueRenderOption + dateTimeRenderOption?: DateTimeRenderOption +} + +// https://developers.google.com/sheets/api/reference/rest/v4/spreadsheets.values/batchGet#response-body +interface BatchGetResponse { + spreadsheetId: string + valueRanges: ValueRange[] +} + interface AppendRequest { range: string params: AppendParams @@ -268,39 +282,57 @@ export class GoogleSheetsMock { } private mockAPI() { - this.get(`/v4/spreadsheets/${this.config.spreadsheetId}/`, () => + const spreadsheetId = this.config.spreadsheetId + + this.get(`/v4/spreadsheets/${spreadsheetId}/`, () => this.handleGetSpreadsheet() ) // https://developers.google.com/sheets/api/reference/rest/v4/spreadsheets.values/batchUpdate this.post( - `/v4/spreadsheets/${this.config.spreadsheetId}/:batchUpdate`, + `/v4/spreadsheets/${spreadsheetId}/:batchUpdate`, (_uri, request) => this.handleBatchUpdate(request as BatchUpdateRequest) ) // https://developers.google.com/sheets/api/reference/rest/v4/spreadsheets.values/update this.put( - new RegExp(`/v4/spreadsheets/${this.config.spreadsheetId}/values/.*`), + new RegExp(`/v4/spreadsheets/${spreadsheetId}/values/.*`), (_uri, request) => this.handleValueUpdate(request as ValueRange) ) - // https://developers.google.com/sheets/api/reference/rest/v4/spreadsheets.values/get + // https://developers.google.com/sheets/api/reference/rest/v4/spreadsheets.values/batchGet this.get( - new RegExp(`/v4/spreadsheets/${this.config.spreadsheetId}/values/.*`), + new RegExp(`/v4/spreadsheets/${spreadsheetId}/values:batchGet.*`), uri => { - const range = uri.split("/").pop() - if (!range) { - throw new Error("No range provided") + const url = new URL(uri, "https://sheets.googleapis.com/") + const params: BatchGetParams = { + ranges: url.searchParams.getAll("ranges"), + majorDimension: + (url.searchParams.get("majorDimension") as Dimension) || "ROWS", + valueRenderOption: + (url.searchParams.get("valueRenderOption") as ValueRenderOption) || + undefined, + dateTimeRenderOption: + (url.searchParams.get( + "dateTimeRenderOption" + ) as DateTimeRenderOption) || undefined, } - return this.getValueRange(decodeURIComponent(range)) + return this.handleBatchGet(params as unknown as BatchGetParams) } ) + // https://developers.google.com/sheets/api/reference/rest/v4/spreadsheets.values/get + this.get(new RegExp(`/v4/spreadsheets/${spreadsheetId}/values/.*`), uri => { + const range = uri.split("/").pop() + if (!range) { + throw new Error("No range provided") + } + return this.getValueRange(decodeURIComponent(range)) + }) + // https://developers.google.com/sheets/api/reference/rest/v4/spreadsheets.values/append this.post( - new RegExp( - `/v4/spreadsheets/${this.config.spreadsheetId}/values/.*:append` - ), + new RegExp(`/v4/spreadsheets/${spreadsheetId}/values/.*:append`), (_uri, request) => { const url = new URL(_uri, "https://sheets.googleapis.com/") const params: Record = Object.fromEntries( @@ -373,6 +405,19 @@ export class GoogleSheetsMock { } } + private handleBatchGet(params: BatchGetParams): BatchGetResponse { + const { ranges, majorDimension } = params + + if (majorDimension && majorDimension !== "ROWS") { + throw new Error("Only row-major updates are supported") + } + + return { + spreadsheetId: this.spreadsheet.spreadsheetId, + valueRanges: ranges.map(range => this.getValueRange(range)), + } + } + private handleBatchUpdate( batchUpdateRequest: BatchUpdateRequest ): BatchUpdateResponse { From 96fbc8fff093fb5d49e6fa5a99441246c45b389e Mon Sep 17 00:00:00 2001 From: Martin McKeaveney Date: Mon, 9 Sep 2024 18:07:47 +0100 Subject: [PATCH 58/85] feature flag support --- packages/backend-core/src/features/index.ts | 1 + .../tests/core/utilities/structures/quotas.ts | 4 +++ .../builder/portal/settings/ai/index.svelte | 4 ++- packages/builder/src/stores/portal/menu.js | 15 ++++++--- packages/types/src/sdk/featureFlag.ts | 1 + .../src/api/controllers/global/configs.ts | 4 +-- .../controllers/global/tests/configs.spec.ts | 31 ++++++++++++++++++- 7 files changed, 52 insertions(+), 8 deletions(-) diff --git a/packages/backend-core/src/features/index.ts b/packages/backend-core/src/features/index.ts index b84237b1f0..f855daf8a5 100644 --- a/packages/backend-core/src/features/index.ts +++ b/packages/backend-core/src/features/index.ts @@ -268,4 +268,5 @@ export class FlagSet, T extends { [key: string]: V }> { export const flags = new FlagSet({ DEFAULT_VALUES: Flag.boolean(env.isDev()), SQS: Flag.boolean(env.isDev()), + AI_CUSTOM_CONFIGS: Flag.boolean(env.isDev()), }) diff --git a/packages/backend-core/tests/core/utilities/structures/quotas.ts b/packages/backend-core/tests/core/utilities/structures/quotas.ts index 8d0b05fe1e..58817e4831 100644 --- a/packages/backend-core/tests/core/utilities/structures/quotas.ts +++ b/packages/backend-core/tests/core/utilities/structures/quotas.ts @@ -17,6 +17,7 @@ export const usage = (users: number = 0, creators: number = 0): QuotaUsage => { automations: 0, dayPasses: 0, queries: 0, + budibaseAICredits: 0, triggers: {}, breakdown: { rowQueries: { @@ -46,12 +47,14 @@ export const usage = (users: number = 0, creators: number = 0): QuotaUsage => { automations: 0, dayPasses: 0, queries: 0, + budibaseAICredits: 0, triggers: {}, }, current: { automations: 0, dayPasses: 0, queries: 0, + budibaseAICredits: 0, triggers: {}, }, }, @@ -62,6 +65,7 @@ export const usage = (users: number = 0, creators: number = 0): QuotaUsage => { creators, userGroups: 0, rows: 0, + aiCustomConfigs: 0, triggers: {}, }, } diff --git a/packages/builder/src/pages/builder/portal/settings/ai/index.svelte b/packages/builder/src/pages/builder/portal/settings/ai/index.svelte index 77d6aa952a..c4403e5045 100644 --- a/packages/builder/src/pages/builder/portal/settings/ai/index.svelte +++ b/packages/builder/src/pages/builder/portal/settings/ai/index.svelte @@ -80,7 +80,9 @@ } async function deleteConfig(key) { - // Delete a configuration + // We don't store the default BB AI config in the DB + delete fullAIConfig.config.budibase_ai + // Delete the configuration delete fullAIConfig.config[key] try { diff --git a/packages/builder/src/stores/portal/menu.js b/packages/builder/src/stores/portal/menu.js index ec40f6d4ee..1e6282894e 100644 --- a/packages/builder/src/stores/portal/menu.js +++ b/packages/builder/src/stores/portal/menu.js @@ -1,7 +1,9 @@ import { derived } from "svelte/store" import { admin } from "./admin" import { auth } from "./auth" +import { isEnabled } from "helpers/featureFlags" import { sdk } from "@budibase/shared-core" +import { FeatureFlag } from "@budibase/types"; export const menu = derived([admin, auth], ([$admin, $auth]) => { const user = $auth?.user @@ -45,10 +47,6 @@ export const menu = derived([admin, auth], ([$admin, $auth]) => { title: "Auth", href: "/builder/portal/settings/auth", }, - { - title: "AI", - href: "/builder/portal/settings/ai", - }, { title: "Email", href: "/builder/portal/settings/email", @@ -66,6 +64,15 @@ export const menu = derived([admin, auth], ([$admin, $auth]) => { href: "/builder/portal/settings/environment", }, ] + if (isEnabled(FeatureFlag.AI_CUSTOM_CONFIGS)) { + settingsSubPages.push( + { + title: "AI", + href: "/builder/portal/settings/ai", + } + ) + } + if (!cloud) { settingsSubPages.push({ title: "Version", diff --git a/packages/types/src/sdk/featureFlag.ts b/packages/types/src/sdk/featureFlag.ts index 257b4ee576..69b2a3b1b2 100644 --- a/packages/types/src/sdk/featureFlag.ts +++ b/packages/types/src/sdk/featureFlag.ts @@ -1,6 +1,7 @@ export enum FeatureFlag { PER_CREATOR_PER_USER_PRICE = "PER_CREATOR_PER_USER_PRICE", PER_CREATOR_PER_USER_PRICE_ALERT = "PER_CREATOR_PER_USER_PRICE_ALERT", + AI_CUSTOM_CONFIGS = "AI_CUSTOM_CONFIGS", } export interface TenantFeatureFlags { diff --git a/packages/worker/src/api/controllers/global/configs.ts b/packages/worker/src/api/controllers/global/configs.ts index b8feb6158f..ffe7a1c717 100644 --- a/packages/worker/src/api/controllers/global/configs.ts +++ b/packages/worker/src/api/controllers/global/configs.ts @@ -135,7 +135,6 @@ const getEventFns = async (config: Config, existing?: Config) => { } } } - return fns } @@ -344,11 +343,12 @@ async function enrichAIConfig(aiConfig: AIConfig) { // Return the Budibase AI data source as part of the response if licensing allows const budibaseAIEnabled = await pro.features.isBudibaseAIEnabled() + const defaultConfigExists = Object.keys(aiConfig.config).some(key => aiConfig.config[key].isDefault) if (budibaseAIEnabled) { aiConfig.config["budibase_ai"] = { provider: "OpenAI", active: true, - isDefault: true, + isDefault: !defaultConfigExists, defaultModel: env.BUDIBASE_AI_DEFAULT_MODEL, name: "Budibase AI", } diff --git a/packages/worker/src/api/controllers/global/tests/configs.spec.ts b/packages/worker/src/api/controllers/global/tests/configs.spec.ts index 2ef00252ff..4bb3d598a6 100644 --- a/packages/worker/src/api/controllers/global/tests/configs.spec.ts +++ b/packages/worker/src/api/controllers/global/tests/configs.spec.ts @@ -1,4 +1,3 @@ -import { expect } from "vitest" import { configs } from "@budibase/backend-core" import { UserCtx } from "@budibase/types" import * as pro from "@budibase/pro" @@ -81,6 +80,35 @@ describe("Global configs controller", () => { }) }) + it("Should not not return the default Budibase AI config when on self host", async () => { + pro.features.isBudibaseAIEnabled = jest.fn(() => false) + configs.getConfig.mockResolvedValue({ + config: { + ai: { + apiKey: "abc123APIKey", + baseUrl: "https://api.example.com", + }, + }, + }) + const ctx = { + params: { + type: "ai", + }, + throw: jest.fn(), + } as UserCtx + + await find(ctx) + + expect(ctx.body).toEqual({ + config: { + ai: { + apiKey: "--secret-value--", + baseUrl: "https://api.example.com", + }, + }, + }) + }) + it("Should not update existing secrets when updating an existing AI Config", async () => { const newConfig = { type: "ai", @@ -114,4 +142,5 @@ describe("Global configs controller", () => { // should be unchanged expect(newConfig.config.aiconfig.apiKey === "myapikey") }) + }) From 8b26ab422f91de19bdc57f5ff5976b9db02d0d38 Mon Sep 17 00:00:00 2001 From: Martin McKeaveney Date: Mon, 9 Sep 2024 18:19:10 +0100 Subject: [PATCH 59/85] remove conditional chain --- .../builder/src/pages/builder/portal/users/users/index.svelte | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/builder/src/pages/builder/portal/users/users/index.svelte b/packages/builder/src/pages/builder/portal/users/users/index.svelte index 91d9c99af6..b91b2129e6 100644 --- a/packages/builder/src/pages/builder/portal/users/users/index.svelte +++ b/packages/builder/src/pages/builder/portal/users/users/index.svelte @@ -127,7 +127,7 @@ name: user.firstName ? user.firstName + " " + user.lastName : "", userGroups, __selectable: - role?.value === Constants.BudibaseRoles.Owner ? false : undefined, + role.value === Constants.BudibaseRoles.Owner ? false : undefined, apps: [...new Set(Object.keys(user.roles))], access: role.sortOrder, } From dc9e1cbbc74633377604109ca109810929d5084e Mon Sep 17 00:00:00 2001 From: Adria Navarro Date: Tue, 10 Sep 2024 10:44:56 +0200 Subject: [PATCH 60/85] Enrich view with all schema --- packages/server/src/sdk/app/views/index.ts | 2 +- packages/types/src/documents/app/table/constants.ts | 5 +++++ packages/types/src/sdk/view.ts | 12 ++---------- 3 files changed, 8 insertions(+), 11 deletions(-) diff --git a/packages/server/src/sdk/app/views/index.ts b/packages/server/src/sdk/app/views/index.ts index 13d81d6802..d7e05abf2f 100644 --- a/packages/server/src/sdk/app/views/index.ts +++ b/packages/server/src/sdk/app/views/index.ts @@ -193,8 +193,8 @@ export async function enrichSchema( const isVisible = !!viewFieldSchema?.visible const isReadonly = !!viewFieldSchema?.readonly result[relTableFieldName] = { + ...relTableField, ...viewFieldSchema, - type: relTableField.type, name: relTableField.name, visible: isVisible, readonly: isReadonly, diff --git a/packages/types/src/documents/app/table/constants.ts b/packages/types/src/documents/app/table/constants.ts index 210ad1423d..fffaddc5df 100644 --- a/packages/types/src/documents/app/table/constants.ts +++ b/packages/types/src/documents/app/table/constants.ts @@ -10,6 +10,11 @@ export enum AutoReason { FOREIGN_KEY = "foreign_key", } +export type FieldSubType = + | AutoFieldSubType + | JsonFieldSubType + | BBReferenceFieldSubType + export enum AutoFieldSubType { CREATED_BY = "createdBy", CREATED_AT = "createdAt", diff --git a/packages/types/src/sdk/view.ts b/packages/types/src/sdk/view.ts index 7480bf563f..422207197d 100644 --- a/packages/types/src/sdk/view.ts +++ b/packages/types/src/sdk/view.ts @@ -1,9 +1,4 @@ -import { - FieldSchema, - FieldType, - RelationSchemaField, - ViewV2, -} from "../documents" +import { FieldSchema, RelationSchemaField, ViewV2 } from "../documents" export interface ViewV2Enriched extends ViewV2 { schema?: { @@ -13,7 +8,4 @@ export interface ViewV2Enriched extends ViewV2 { } } -export interface ViewV2ColumnEnriched extends RelationSchemaField { - name: string - type: FieldType -} +export type ViewV2ColumnEnriched = RelationSchemaField & FieldSchema From 7a376d96295b1fd3d8d0367e2faf7d898c0dce53 Mon Sep 17 00:00:00 2001 From: Martin McKeaveney Date: Tue, 10 Sep 2024 10:26:18 +0100 Subject: [PATCH 61/85] Update packages/worker/src/api/controllers/global/configs.ts Co-authored-by: Sam Rose --- packages/worker/src/api/controllers/global/configs.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/worker/src/api/controllers/global/configs.ts b/packages/worker/src/api/controllers/global/configs.ts index ffe7a1c717..5345c25bc2 100644 --- a/packages/worker/src/api/controllers/global/configs.ts +++ b/packages/worker/src/api/controllers/global/configs.ts @@ -335,7 +335,7 @@ function enrichOIDCLogos(oidcLogos: OIDCLogosConfig) { async function enrichAIConfig(aiConfig: AIConfig) { // Strip out the API Keys from the response so they don't show in the UI - for (let key in aiConfig.config) { + for (const key in aiConfig.config) { if (aiConfig.config[key].apiKey) { aiConfig.config[key].apiKey = PASSWORD_REPLACEMENT } From 4ff0dab399440429b14f8886ea37e3ada27c8455 Mon Sep 17 00:00:00 2001 From: Sam Rose Date: Tue, 10 Sep 2024 10:51:22 +0100 Subject: [PATCH 62/85] Respond to PR feedback. --- .../server/src/integrations/googlesheets.ts | 11 +- .../integrations/tests/utils/googlesheets.ts | 221 +++++++++++------- 2 files changed, 140 insertions(+), 92 deletions(-) diff --git a/packages/server/src/integrations/googlesheets.ts b/packages/server/src/integrations/googlesheets.ts index 960d01ab52..af3212bf60 100644 --- a/packages/server/src/integrations/googlesheets.ts +++ b/packages/server/src/integrations/googlesheets.ts @@ -379,6 +379,8 @@ export class GoogleSheetsIntegration implements DatasourcePlus { case Operation.BULK_CREATE: return this.createBulk({ sheet, rows: json.body as Row[] }) case Operation.BULK_UPSERT: + // This is technically not correct because it won't update existing + // rows, but it's better than not having this functionality at all. return this.createBulk({ sheet, rows: json.body as Row[] }) case Operation.READ: return this.read({ ...json, sheet }) @@ -397,14 +399,19 @@ export class GoogleSheetsIntegration implements DatasourcePlus { sheet, }) case Operation.CREATE_TABLE: - if (json.table === undefined) { + if (json.table == null) { throw new Error( "attempted to create a table without specifying the table to create" ) } return this.createTable(json.table) case Operation.UPDATE_TABLE: - return this.updateTable(json.table!) + if (json.table == null) { + throw new Error( + "attempted to create a table without specifying the table to create" + ) + } + return this.updateTable(json.table) case Operation.DELETE_TABLE: return this.deleteTable(json?.table?.name) default: diff --git a/packages/server/src/integrations/tests/utils/googlesheets.ts b/packages/server/src/integrations/tests/utils/googlesheets.ts index 4851104c8b..0f92fea6cd 100644 --- a/packages/server/src/integrations/tests/utils/googlesheets.ts +++ b/packages/server/src/integrations/tests/utils/googlesheets.ts @@ -10,31 +10,71 @@ import { Datasource } from "@budibase/types" import nock from "nock" import { GoogleSheetsConfig } from "../../googlesheets" +import type { + SpreadsheetProperties, + ExtendedValue, + WorksheetDimension, + WorksheetDimensionProperties, + WorksheetProperties, + CellData, + CellBorder, + CellFormat, + CellPadding, + Color, +} from "google-spreadsheet/src/lib/types/sheets-types" + +const BLACK: Color = { red: 0, green: 0, blue: 0 } +const WHITE: Color = { red: 1, green: 1, blue: 1 } +const NO_PADDING: CellPadding = { top: 0, right: 0, bottom: 0, left: 0 } +const DEFAULT_BORDER: CellBorder = { + style: "SOLID", + width: 1, + color: BLACK, + colorStyle: { rgbColor: BLACK }, +} +const DEFAULT_CELL_FORMAT: CellFormat = { + hyperlinkDisplayType: "PLAIN_TEXT", + horizontalAlignment: "LEFT", + verticalAlignment: "BOTTOM", + wrapStrategy: "OVERFLOW_CELL", + textDirection: "LEFT_TO_RIGHT", + textRotation: { angle: 0, vertical: false }, + padding: NO_PADDING, + backgroundColorStyle: { rgbColor: BLACK }, + borders: { + top: DEFAULT_BORDER, + bottom: DEFAULT_BORDER, + left: DEFAULT_BORDER, + right: DEFAULT_BORDER, + }, + numberFormat: { + type: "NUMBER", + pattern: "General", + }, + backgroundColor: WHITE, + textFormat: { + foregroundColor: BLACK, + fontFamily: "Arial", + fontSize: 10, + bold: false, + italic: false, + strikethrough: false, + underline: false, + }, +} // https://protobuf.dev/reference/protobuf/google.protobuf/#value type Value = string | number | boolean | null -// https://developers.google.com/sheets/api/reference/rest/v4/Dimension -type Dimension = "ROWS" | "COLUMNS" - interface Range { row: number column: number } -// https://developers.google.com/sheets/api/reference/rest/v4/spreadsheets/sheets#DimensionProperties -interface DimensionProperties { - hiddenByFilter: boolean - hiddenByUser: boolean - pixelSize: number - // developerMetadata: DeveloperMetadata[] - // dataSourceColumnReference: DataSourceColumnReference -} - // https://developers.google.com/sheets/api/reference/rest/v4/spreadsheets.values#ValueRange interface ValueRange { range: string - majorDimension: Dimension + majorDimension: WorksheetDimension values: Value[][] } @@ -50,41 +90,21 @@ interface UpdateValuesResponse { // https://developers.google.com/sheets/api/reference/rest/v4/spreadsheets/response#AddSheetResponse interface AddSheetResponse { - properties: SheetProperties -} - -interface Response { - addSheet?: AddSheetResponse + properties: WorksheetProperties } // https://developers.google.com/sheets/api/reference/rest/v4/spreadsheets/response interface BatchUpdateResponse { spreadsheetId: string - replies: Response[] + replies: { + addSheet?: AddSheetResponse + }[] updatedSpreadsheet: Spreadsheet } -// https://developers.google.com/sheets/api/reference/rest/v4/spreadsheets/sheets#GridProperties -interface GridProperties { - rowCount: number - columnCount: number - frozenRowCount: number - frozenColumnCount: number - hideGridlines: boolean - rowGroupControlAfter: boolean - columnGroupControlAfter: boolean -} - -// https://developers.google.com/sheets/api/reference/rest/v4/spreadsheets/sheets#SheetProperties -interface SheetProperties { - sheetId: number - title: string - gridProperties: GridProperties -} - // https://developers.google.com/sheets/api/reference/rest/v4/spreadsheets/request#AddSheetRequest interface AddSheetRequest { - properties: SheetProperties + properties: WorksheetProperties } interface Request { @@ -99,26 +119,6 @@ interface BatchUpdateRequest { responseIncludeGridData: boolean } -// https://developers.google.com/sheets/api/reference/rest/v4/spreadsheets/other#ErrorValue -interface ErrorValue { - type: string - message: string -} - -// https://developers.google.com/sheets/api/reference/rest/v4/spreadsheets/other#ExtendedValue -interface ExtendedValue { - stringValue?: string - numberValue?: number - boolValue?: boolean - formulaValue?: string - errorValue?: ErrorValue -} - -// https://developers.google.com/sheets/api/reference/rest/v4/spreadsheets/cells#CellData -interface CellData { - userEnteredValue: ExtendedValue -} - // https://developers.google.com/sheets/api/reference/rest/v4/spreadsheets/sheets#RowData interface RowData { values: CellData[] @@ -129,21 +129,16 @@ interface GridData { startRow: number startColumn: number rowData: RowData[] - rowMetadata: DimensionProperties[] - columnMetadata: DimensionProperties[] + rowMetadata: WorksheetDimensionProperties[] + columnMetadata: WorksheetDimensionProperties[] } // https://developers.google.com/sheets/api/reference/rest/v4/spreadsheets/sheets#Sheet interface Sheet { - properties: SheetProperties + properties: WorksheetProperties data: GridData[] } -// https://developers.google.com/sheets/api/reference/rest/v4/spreadsheets#SpreadsheetProperties -interface SpreadsheetProperties { - title: string -} - // https://developers.google.com/sheets/api/reference/rest/v4/spreadsheets#Spreadsheet interface Spreadsheet { properties: SpreadsheetProperties @@ -178,7 +173,7 @@ interface AppendParams { // https://developers.google.com/sheets/api/reference/rest/v4/spreadsheets.values/batchGet#query-parameters interface BatchGetParams { ranges: string[] - majorDimension?: Dimension + majorDimension?: WorksheetDimension valueRenderOption?: ValueRenderOption dateTimeRenderOption?: DateTimeRenderOption } @@ -215,6 +210,12 @@ export class GoogleSheetsMock { this.spreadsheet = { properties: { title: "Test Spreadsheet", + locale: "en_US", + autoRecalc: "ON_CHANGE", + timeZone: "America/New_York", + defaultFormat: {}, + iterativeCalculationSettings: {}, + spreadsheetTheme: {}, }, spreadsheetId: config.spreadsheetId, sheets: [], @@ -308,7 +309,8 @@ export class GoogleSheetsMock { const params: BatchGetParams = { ranges: url.searchParams.getAll("ranges"), majorDimension: - (url.searchParams.get("majorDimension") as Dimension) || "ROWS", + (url.searchParams.get("majorDimension") as WorksheetDimension) || + "ROWS", valueRenderOption: (url.searchParams.get("valueRenderOption") as ValueRenderOption) || undefined, @@ -377,8 +379,11 @@ export class GoogleSheetsMock { hiddenByUser: false, hiddenByFilter: false, pixelSize: 100, + developerMetadata: [], }) + // It's important to give back a correct updated range because the API + // library we use makes use of it to assign the correct row IDs to rows. const updatedRange = this.createA1FromRanges( sheet, { @@ -421,24 +426,31 @@ export class GoogleSheetsMock { private handleBatchUpdate( batchUpdateRequest: BatchUpdateRequest ): BatchUpdateResponse { - const replies: Response[] = [] + const response: BatchUpdateResponse = { + spreadsheetId: this.spreadsheet.spreadsheetId, + replies: [], + updatedSpreadsheet: this.spreadsheet, + } for (const request of batchUpdateRequest.requests) { if (request.addSheet) { - const response = this.handleAddSheet(request.addSheet) - replies.push({ addSheet: response }) + response.replies.push({ + addSheet: this.handleAddSheet(request.addSheet), + }) } } - return { - spreadsheetId: this.spreadsheet.spreadsheetId, - replies, - updatedSpreadsheet: this.spreadsheet, - } + return response } private handleAddSheet(request: AddSheetRequest): AddSheetResponse { - const properties: SheetProperties = { + const properties: Omit = { + index: this.spreadsheet.sheets.length, + hidden: false, + rightToLeft: false, + tabColor: BLACK, + tabColorStyle: { rgbColor: BLACK }, + sheetType: "GRID", title: request.properties.title, sheetId: this.spreadsheet.sheets.length, gridProperties: { @@ -453,11 +465,13 @@ export class GoogleSheetsMock { } this.spreadsheet.sheets.push({ - properties, + properties: properties as WorksheetProperties, data: [this.createEmptyGrid(100, 26)], }) - return { properties } + // dataSourceSheetProperties is only returned by the API if the sheet type is + // DATA_SOURCE, which we aren't using, so sadly we need to cast here. + return { properties: properties as WorksheetProperties } } private handleGetSpreadsheet(): Spreadsheet { @@ -518,7 +532,7 @@ export class GoogleSheetsMock { if (!cell) { throw new Error("Cell not found") } - values.push(this.unwrapValue(cell.userEnteredValue)) + values.push(this.cellValue(cell)) } valueRange.values.push(values) } @@ -529,28 +543,32 @@ export class GoogleSheetsMock { private valuesToRowData(values: Value[]): RowData { return { values: values.map(v => { - return { userEnteredValue: this.createValue(v) } + return this.createCellData(v) }), } } private unwrapValue(from: ExtendedValue): Value { - if (from.stringValue !== undefined) { + if ("stringValue" in from) { return from.stringValue - } else if (from.numberValue !== undefined) { + } else if ("numberValue" in from) { return from.numberValue - } else if (from.boolValue !== undefined) { + } else if ("boolValue" in from) { return from.boolValue - } else if (from.formulaValue !== undefined) { + } else if ("formulaValue" in from) { return from.formulaValue } else { return null } } + private cellValue(from: CellData): Value { + return this.unwrapValue(from.userEnteredValue) + } + private createValue(from: Value): ExtendedValue { if (from == null) { - return {} + return {} as ExtendedValue } else if (typeof from === "string") { return { stringValue: from, @@ -568,29 +586,52 @@ export class GoogleSheetsMock { } } + /** + * Because the structure of a CellData is very nested and contains a lot of + * extraneous formatting information, this function abstracts it away and just + * lets you create a cell containing a given value. + * + * When you want to read the value back out, use {@link cellValue}. + * + * @param value value to store in the returned cell + * @returns a CellData containing the given value. Read it back out with + * {@link cellValue} + */ + private createCellData(value: Value): CellData { + return { + userEnteredValue: this.createValue(value), + effectiveValue: this.createValue(value), + formattedValue: value?.toString() || "", + userEnteredFormat: DEFAULT_CELL_FORMAT, + effectiveFormat: DEFAULT_CELL_FORMAT, + } + } + private createEmptyGrid(numRows: number, numCols: number): GridData { const rowData: RowData[] = [] for (let row = 0; row < numRows; row++) { const cells: CellData[] = [] for (let col = 0; col < numCols; col++) { - cells.push({ userEnteredValue: this.createValue(null) }) + cells.push(this.createCellData(null)) } rowData.push({ values: cells }) } - const rowMetadata: DimensionProperties[] = [] + const rowMetadata: WorksheetDimensionProperties[] = [] for (let row = 0; row < numRows; row++) { rowMetadata.push({ hiddenByFilter: false, hiddenByUser: false, pixelSize: 100, + developerMetadata: [], }) } - const columnMetadata: DimensionProperties[] = [] + const columnMetadata: WorksheetDimensionProperties[] = [] for (let col = 0; col < numCols; col++) { columnMetadata.push({ hiddenByFilter: false, hiddenByUser: false, pixelSize: 100, + developerMetadata: [], }) } @@ -616,7 +657,7 @@ export class GoogleSheetsMock { if (!cellData) { return undefined } - return this.unwrapValue(cellData.userEnteredValue) + return this.cellValue(cellData) } private getCellNumericIndexes( From 4fed5787ac1a11d47ebd82fa3c69c7accc3cec47 Mon Sep 17 00:00:00 2001 From: Martin McKeaveney Date: Tue, 10 Sep 2024 10:54:50 +0100 Subject: [PATCH 63/85] review comments --- .../portal/settings/ai/AIConfigTile.svelte | 17 ++++++++--------- .../builder/portal/settings/ai/index.svelte | 6 ++++-- .../src/api/controllers/global/configs.ts | 8 ++++---- .../controllers/global/tests/configs.spec.ts | 4 ++-- 4 files changed, 18 insertions(+), 17 deletions(-) diff --git a/packages/builder/src/pages/builder/portal/settings/ai/AIConfigTile.svelte b/packages/builder/src/pages/builder/portal/settings/ai/AIConfigTile.svelte index d4d5279d3e..693fbe7431 100644 --- a/packages/builder/src/pages/builder/portal/settings/ai/AIConfigTile.svelte +++ b/packages/builder/src/pages/builder/portal/settings/ai/AIConfigTile.svelte @@ -6,6 +6,13 @@ import TogetherAILogo from "./logos/TogetherAI.svelte" import { Providers } from "./constants" + const logos = { + ["Budibase AI"]: BudibaseLogo, + [Providers.OpenAI.name]: OpenAILogo, + [Providers.Anthropic.name]: AnthropicLogo, + [Providers.TogetherAI.name]: TogetherAILogo, + } + export let config export let disabled @@ -17,15 +24,7 @@
- {#if config.name === "Budibase AI"} - - {:else if config.provider === Providers.OpenAI.name} - - {:else if config.provider === Providers.Anthropic.name} - - {:else if config.provider === Providers.TogetherAI.name} - - {/if} +
{config.provider} diff --git a/packages/builder/src/pages/builder/portal/settings/ai/index.svelte b/packages/builder/src/pages/builder/portal/settings/ai/index.svelte index c4403e5045..2ac1609e7c 100644 --- a/packages/builder/src/pages/builder/portal/settings/ai/index.svelte +++ b/packages/builder/src/pages/builder/portal/settings/ai/index.svelte @@ -69,13 +69,14 @@ try { await API.saveConfig(fullAIConfig) notifications.success(`Successfully saved and activated AI Configuration`) - await fetchAIConfig() } catch (error) { notifications.error( `Failed to save AI Configuration, reason: ${ error?.message || "Unknown" }` ) + } finally { + await fetchAIConfig() } } @@ -88,11 +89,12 @@ try { await API.saveConfig(fullAIConfig) notifications.success(`Deleted config`) - await fetchAIConfig() } catch (error) { notifications.error( `Failed to delete config, reason: ${error?.message || "Unknown"}` ) + } finally { + await fetchAIConfig() } } diff --git a/packages/worker/src/api/controllers/global/configs.ts b/packages/worker/src/api/controllers/global/configs.ts index ffe7a1c717..95e0ba471f 100644 --- a/packages/worker/src/api/controllers/global/configs.ts +++ b/packages/worker/src/api/controllers/global/configs.ts @@ -211,10 +211,8 @@ async function verifyOIDCConfig(config: OIDCConfigs) { export async function verifyAIConfig( config: AIConfig, - existingConfig?: AIConfig + existingConfig: AIConfig ) { - if (!existingConfig) return - // ensure that the redacted API keys are not overwritten in the DB for (let uuid in existingConfig.config) { if (config[uuid]?.apiKey === PASSWORD_REPLACEMENT) { @@ -251,7 +249,9 @@ export async function save(ctx: UserCtx) { await verifyOIDCConfig(config) break case ConfigType.AI: - await verifyAIConfig(config, existingConfig) + if (existingConfig) { + await verifyAIConfig(config, existingConfig) + } break } } catch (err: any) { diff --git a/packages/worker/src/api/controllers/global/tests/configs.spec.ts b/packages/worker/src/api/controllers/global/tests/configs.spec.ts index 4bb3d598a6..8b859dcb89 100644 --- a/packages/worker/src/api/controllers/global/tests/configs.spec.ts +++ b/packages/worker/src/api/controllers/global/tests/configs.spec.ts @@ -45,7 +45,7 @@ describe("Global configs controller", () => { }) it("Should return the default BB AI config when the feature is turned on", async () => { - pro.features.isBudibaseAIEnabled = jest.fn(() => true) + jest.spyOn(pro.features, "isBudibaseAIEnabled").mockImplementation(() => true) configs.getConfig.mockResolvedValue({ config: { ai: { @@ -81,7 +81,7 @@ describe("Global configs controller", () => { }) it("Should not not return the default Budibase AI config when on self host", async () => { - pro.features.isBudibaseAIEnabled = jest.fn(() => false) + jest.spyOn(pro.features, "isBudibaseAIEnabled").mockImplementation(() => false) configs.getConfig.mockResolvedValue({ config: { ai: { From 7438a1d65cdaf888e7d11677d8d3e6e48982902a Mon Sep 17 00:00:00 2001 From: Sam Rose Date: Tue, 10 Sep 2024 10:59:11 +0100 Subject: [PATCH 64/85] tidy up null checks --- packages/server/src/integrations/googlesheets.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/packages/server/src/integrations/googlesheets.ts b/packages/server/src/integrations/googlesheets.ts index af3212bf60..0d766ac1ef 100644 --- a/packages/server/src/integrations/googlesheets.ts +++ b/packages/server/src/integrations/googlesheets.ts @@ -399,14 +399,14 @@ export class GoogleSheetsIntegration implements DatasourcePlus { sheet, }) case Operation.CREATE_TABLE: - if (json.table == null) { + if (!json.table) { throw new Error( "attempted to create a table without specifying the table to create" ) } return this.createTable(json.table) case Operation.UPDATE_TABLE: - if (json.table == null) { + if (!json.table) { throw new Error( "attempted to create a table without specifying the table to create" ) From f63c95e44c282743b31b80a0306feeeb90e30452 Mon Sep 17 00:00:00 2001 From: mike12345567 Date: Tue, 10 Sep 2024 13:59:21 +0100 Subject: [PATCH 65/85] Adding SQL_MAX_RELATED_ROWS environment variable, defaults to 500, allows for 500 rows per relationship. --- packages/backend-core/src/environment.ts | 1 + packages/backend-core/src/sql/sql.ts | 9 ++++++++- 2 files changed, 9 insertions(+), 1 deletion(-) diff --git a/packages/backend-core/src/environment.ts b/packages/backend-core/src/environment.ts index 3d931a8c67..6bef6efeb3 100644 --- a/packages/backend-core/src/environment.ts +++ b/packages/backend-core/src/environment.ts @@ -171,6 +171,7 @@ const environment = { // Couch/search SQL_LOGGING_ENABLE: process.env.SQL_LOGGING_ENABLE, SQL_MAX_ROWS: process.env.SQL_MAX_ROWS, + SQL_MAX_RELATED_ROWS: process.env.MAX_RELATED_ROWS, // smtp SMTP_FALLBACK_ENABLED: process.env.SMTP_FALLBACK_ENABLED, SMTP_USER: process.env.SMTP_USER, diff --git a/packages/backend-core/src/sql/sql.ts b/packages/backend-core/src/sql/sql.ts index b1a2bc060a..d8ad791829 100644 --- a/packages/backend-core/src/sql/sql.ts +++ b/packages/backend-core/src/sql/sql.ts @@ -49,6 +49,13 @@ function getBaseLimit() { return envLimit || 5000 } +function getRelationshipLimit() { + const envLimit = environment.SQL_MAX_RELATED_ROWS + ? parseInt(environment.SQL_MAX_RELATED_ROWS) + : null + return envLimit || 500 +} + function getTableName(table?: Table): string | undefined { // SQS uses the table ID rather than the table name if ( @@ -921,7 +928,7 @@ class InternalBuilder { const primaryKey = `${toAlias}.${toPrimary || toKey}` let subQuery: Knex.QueryBuilder = knex .from(toTableWithSchema) - .limit(getBaseLimit()) + .limit(getRelationshipLimit()) // add sorting to get consistent order .orderBy(primaryKey) From 5f36a51e8cd585051c12c43e43cd4c63477cd186 Mon Sep 17 00:00:00 2001 From: Sam Rose Date: Tue, 10 Sep 2024 14:39:23 +0100 Subject: [PATCH 66/85] Add new test creating multiple tables. --- .../integrations/tests/googlesheets.spec.ts | 46 ++++++++++++++++++- 1 file changed, 45 insertions(+), 1 deletion(-) diff --git a/packages/server/src/integrations/tests/googlesheets.spec.ts b/packages/server/src/integrations/tests/googlesheets.spec.ts index d3044d17eb..4db85a80c2 100644 --- a/packages/server/src/integrations/tests/googlesheets.spec.ts +++ b/packages/server/src/integrations/tests/googlesheets.spec.ts @@ -53,7 +53,7 @@ describe("Google Sheets Integration", () => { describe("create", () => { it("creates a new table", async () => { - await config.api.table.save({ + const table = await config.api.table.save({ name: "Test Table", type: "table", sourceId: datasource._id!, @@ -76,11 +76,55 @@ describe("Google Sheets Integration", () => { }, }) + expect(table.name).toEqual("Test Table") + expect(mock.cell("A1")).toEqual("name") expect(mock.cell("B1")).toEqual("description") expect(mock.cell("A2")).toEqual(null) expect(mock.cell("B2")).toEqual(null) }) + + it("can handle multiple tables", async () => { + const table1 = await config.api.table.save({ + name: "Test Table 1", + type: "table", + sourceId: datasource._id!, + sourceType: TableSourceType.EXTERNAL, + schema: { + one: { + name: "one", + type: FieldType.STRING, + constraints: { + type: "string", + }, + }, + }, + }) + + const table2 = await config.api.table.save({ + name: "Test Table 2", + type: "table", + sourceId: datasource._id!, + sourceType: TableSourceType.EXTERNAL, + schema: { + two: { + name: "two", + type: FieldType.STRING, + constraints: { + type: "string", + }, + }, + }, + }) + + expect(table1.name).toEqual("Test Table 1") + expect(table2.name).toEqual("Test Table 2") + + expect(mock.cell("Test Table 1!A1")).toEqual("one") + expect(mock.cell("Test Table 1!A2")).toEqual(null) + expect(mock.cell("Test Table 2!A1")).toEqual("two") + expect(mock.cell("Test Table 2!A2")).toEqual(null) + }) }) describe("update", () => { From 933ca952d2e26ade754c9973099b676265774ec1 Mon Sep 17 00:00:00 2001 From: Martin McKeaveney Date: Tue, 10 Sep 2024 14:50:17 +0100 Subject: [PATCH 67/85] lint --- .../builder/portal/settings/ai/AIConfigTile.svelte | 6 +++++- packages/builder/src/stores/portal/menu.js | 12 +++++------- .../worker/src/api/controllers/global/configs.ts | 4 +++- .../src/api/controllers/global/tests/configs.spec.ts | 11 +++++++---- 4 files changed, 20 insertions(+), 13 deletions(-) diff --git a/packages/builder/src/pages/builder/portal/settings/ai/AIConfigTile.svelte b/packages/builder/src/pages/builder/portal/settings/ai/AIConfigTile.svelte index 693fbe7431..2907919ce7 100644 --- a/packages/builder/src/pages/builder/portal/settings/ai/AIConfigTile.svelte +++ b/packages/builder/src/pages/builder/portal/settings/ai/AIConfigTile.svelte @@ -24,7 +24,11 @@
- +
{config.provider} diff --git a/packages/builder/src/stores/portal/menu.js b/packages/builder/src/stores/portal/menu.js index 1e6282894e..a572539863 100644 --- a/packages/builder/src/stores/portal/menu.js +++ b/packages/builder/src/stores/portal/menu.js @@ -3,7 +3,7 @@ import { admin } from "./admin" import { auth } from "./auth" import { isEnabled } from "helpers/featureFlags" import { sdk } from "@budibase/shared-core" -import { FeatureFlag } from "@budibase/types"; +import { FeatureFlag } from "@budibase/types" export const menu = derived([admin, auth], ([$admin, $auth]) => { const user = $auth?.user @@ -65,12 +65,10 @@ export const menu = derived([admin, auth], ([$admin, $auth]) => { }, ] if (isEnabled(FeatureFlag.AI_CUSTOM_CONFIGS)) { - settingsSubPages.push( - { - title: "AI", - href: "/builder/portal/settings/ai", - } - ) + settingsSubPages.push({ + title: "AI", + href: "/builder/portal/settings/ai", + }) } if (!cloud) { diff --git a/packages/worker/src/api/controllers/global/configs.ts b/packages/worker/src/api/controllers/global/configs.ts index a8dfbe6360..d51ae65c51 100644 --- a/packages/worker/src/api/controllers/global/configs.ts +++ b/packages/worker/src/api/controllers/global/configs.ts @@ -343,7 +343,9 @@ async function enrichAIConfig(aiConfig: AIConfig) { // Return the Budibase AI data source as part of the response if licensing allows const budibaseAIEnabled = await pro.features.isBudibaseAIEnabled() - const defaultConfigExists = Object.keys(aiConfig.config).some(key => aiConfig.config[key].isDefault) + const defaultConfigExists = Object.keys(aiConfig.config).some( + key => aiConfig.config[key].isDefault + ) if (budibaseAIEnabled) { aiConfig.config["budibase_ai"] = { provider: "OpenAI", diff --git a/packages/worker/src/api/controllers/global/tests/configs.spec.ts b/packages/worker/src/api/controllers/global/tests/configs.spec.ts index 8b859dcb89..419c12ccb0 100644 --- a/packages/worker/src/api/controllers/global/tests/configs.spec.ts +++ b/packages/worker/src/api/controllers/global/tests/configs.spec.ts @@ -45,7 +45,9 @@ describe("Global configs controller", () => { }) it("Should return the default BB AI config when the feature is turned on", async () => { - jest.spyOn(pro.features, "isBudibaseAIEnabled").mockImplementation(() => true) + jest + .spyOn(pro.features, "isBudibaseAIEnabled") + .mockImplementation(() => true) configs.getConfig.mockResolvedValue({ config: { ai: { @@ -81,7 +83,9 @@ describe("Global configs controller", () => { }) it("Should not not return the default Budibase AI config when on self host", async () => { - jest.spyOn(pro.features, "isBudibaseAIEnabled").mockImplementation(() => false) + jest + .spyOn(pro.features, "isBudibaseAIEnabled") + .mockImplementation(() => false) configs.getConfig.mockResolvedValue({ config: { ai: { @@ -140,7 +144,6 @@ describe("Global configs controller", () => { await verifyAIConfig(newConfig, existingConfig) // should be unchanged - expect(newConfig.config.aiconfig.apiKey === "myapikey") + expect(newConfig.config.aiconfig.apiKey).toEqual("myapikey") }) - }) From 44a76d67309a64dd88d657731915e44dc133e7fd Mon Sep 17 00:00:00 2001 From: Martin McKeaveney Date: Tue, 10 Sep 2024 15:12:15 +0100 Subject: [PATCH 68/85] add anthropic baseURL --- .../builder/portal/settings/ai/{constants.ts => constants.js} | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) rename packages/builder/src/pages/builder/portal/settings/ai/{constants.ts => constants.js} (97%) diff --git a/packages/builder/src/pages/builder/portal/settings/ai/constants.ts b/packages/builder/src/pages/builder/portal/settings/ai/constants.js similarity index 97% rename from packages/builder/src/pages/builder/portal/settings/ai/constants.ts rename to packages/builder/src/pages/builder/portal/settings/ai/constants.js index 7ff43582e0..deb647f90a 100644 --- a/packages/builder/src/pages/builder/portal/settings/ai/constants.ts +++ b/packages/builder/src/pages/builder/portal/settings/ai/constants.js @@ -44,7 +44,7 @@ export const ConfigMap = { }, Anthropic: { name: "Anthropic", - baseUrl: "", + baseUrl: "https://api.anthropic.com/v1", }, TogetherAI: { name: "TogetherAI", From 2eb53c018ae1bd19c397a52a758e4713ab160f24 Mon Sep 17 00:00:00 2001 From: Martin McKeaveney Date: Tue, 10 Sep 2024 15:19:41 +0100 Subject: [PATCH 69/85] sort settings pages --- packages/builder/src/stores/portal/menu.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/builder/src/stores/portal/menu.js b/packages/builder/src/stores/portal/menu.js index a572539863..19b190fd24 100644 --- a/packages/builder/src/stores/portal/menu.js +++ b/packages/builder/src/stores/portal/menu.js @@ -84,7 +84,7 @@ export const menu = derived([admin, auth], ([$admin, $auth]) => { menu.push({ title: "Settings", href: "/builder/portal/settings", - subPages: settingsSubPages, + subPages: [...settingsSubPages].sort((a, b) => a.title.localeCompare(b.title)), }) } From 2fd5c1a99f9b976793bb1830f1fa7ee031db1bd5 Mon Sep 17 00:00:00 2001 From: mike12345567 Date: Tue, 10 Sep 2024 15:45:32 +0100 Subject: [PATCH 70/85] Handling JSON types within relationships, they need to be parsed as well. --- .../src/api/controllers/row/utils/basic.ts | 54 +++++++++---------- .../src/api/controllers/row/utils/utils.ts | 24 +++------ .../src/sdk/app/rows/search/internal/sqs.ts | 25 +++++---- 3 files changed, 47 insertions(+), 56 deletions(-) diff --git a/packages/server/src/api/controllers/row/utils/basic.ts b/packages/server/src/api/controllers/row/utils/basic.ts index b754e288ed..596cc90b20 100644 --- a/packages/server/src/api/controllers/row/utils/basic.ts +++ b/packages/server/src/api/controllers/row/utils/basic.ts @@ -1,5 +1,5 @@ // need to handle table name + field or just field, depending on if relationships used -import { FieldSchema, FieldType, Row, Table } from "@budibase/types" +import { FieldSchema, FieldType, Row, Table, JsonTypes } from "@budibase/types" import { helpers, PROTECTED_EXTERNAL_COLUMNS, @@ -62,6 +62,22 @@ export function generateIdForRow( return generateRowIdField(idParts) } +function fixJsonTypes(row: Row, table: Table) { + for (let [fieldName, schema] of Object.entries(table.schema)) { + if (JsonTypes.includes(schema.type) && typeof row[fieldName] === "string") { + try { + row[fieldName] = JSON.parse(row[fieldName]) + } catch (err) { + if (!helpers.schema.isDeprecatedSingleUserColumn(schema)) { + // couldn't convert back to array, ignore + delete row[fieldName] + } + } + } + } + return row +} + export function basicProcessing({ row, table, @@ -136,12 +152,15 @@ export function basicProcessing({ const sortField = relatedTable.primaryDisplay || relatedTable.primary![0]! thisRow[col] = (thisRow[col] as Row[]) - .map(relatedRow => { - relatedRow._id = relatedRow._id - ? relatedRow._id - : generateIdForRow(relatedRow, relatedTable) - return relatedRow - }) + .map(relatedRow => + basicProcessing({ + row: relatedRow, + table: relatedTable, + tables, + isLinked: false, + sqs, + }) + ) .sort((a, b) => { const aField = a?.[sortField], bField = b?.[sortField] @@ -157,24 +176,5 @@ export function basicProcessing({ } } } - return thisRow -} - -export function fixArrayTypes(row: Row, table: Table) { - for (let [fieldName, schema] of Object.entries(table.schema)) { - if ( - [FieldType.ARRAY, FieldType.BB_REFERENCE].includes(schema.type) && - typeof row[fieldName] === "string" - ) { - try { - row[fieldName] = JSON.parse(row[fieldName]) - } catch (err) { - if (!helpers.schema.isDeprecatedSingleUserColumn(schema)) { - // couldn't convert back to array, ignore - delete row[fieldName] - } - } - } - } - return row + return fixJsonTypes(thisRow, table) } diff --git a/packages/server/src/api/controllers/row/utils/utils.ts b/packages/server/src/api/controllers/row/utils/utils.ts index 45f0cef085..ac305e70b6 100644 --- a/packages/server/src/api/controllers/row/utils/utils.ts +++ b/packages/server/src/api/controllers/row/utils/utils.ts @@ -14,12 +14,7 @@ import { processFormulas, } from "../../../../utilities/rowProcessor" import { isKnexEmptyReadResponse } from "./sqlUtils" -import { - basicProcessing, - generateIdForRow, - fixArrayTypes, - getInternalRowId, -} from "./basic" +import { basicProcessing, generateIdForRow, getInternalRowId } from "./basic" import sdk from "../../../../sdk" import { processStringSync } from "@budibase/string-templates" import validateJs from "validate.js" @@ -149,16 +144,13 @@ export async function sqlOutputProcessing( rowId = generateIdForRow(row, table) row._id = rowId } - const thisRow = fixArrayTypes( - basicProcessing({ - row, - table, - tables: Object.values(tables), - isLinked: false, - sqs: opts?.sqs, - }), - table - ) + const thisRow = basicProcessing({ + row, + table, + tables: Object.values(tables), + isLinked: false, + sqs: opts?.sqs, + }) if (thisRow._id == null) { throw new Error("Unable to generate row ID for SQL rows") } diff --git a/packages/server/src/sdk/app/rows/search/internal/sqs.ts b/packages/server/src/sdk/app/rows/search/internal/sqs.ts index b5e25e02ea..cf540a244b 100644 --- a/packages/server/src/sdk/app/rows/search/internal/sqs.ts +++ b/packages/server/src/sdk/app/rows/search/internal/sqs.ts @@ -18,6 +18,7 @@ import { } from "@budibase/types" import { buildInternalRelationships, + fixJsonTypes, sqlOutputProcessing, } from "../../../../../api/controllers/row/utils" import sdk from "../../../../index" @@ -182,11 +183,20 @@ function buildTableMap(tables: Table[]) { return tableMap } -function reverseUserColumnMapping(rows: Row[]) { +// table is only needed to handle relationships +function reverseUserColumnMapping(rows: Row[], table?: Table) { const prefixLength = USER_COLUMN_PREFIX.length return rows.map(row => { const finalRow: Row = {} for (let key of Object.keys(row)) { + // handle relationships + if ( + table?.schema[key]?.type === FieldType.LINK && + typeof row[key] === "string" + ) { + // no table required, relationship rows don't contain relationships + row[key] = reverseUserColumnMapping(JSON.parse(row[key])) + } // it should be the first prefix const index = key.indexOf(USER_COLUMN_PREFIX) if (index !== -1) { @@ -261,7 +271,7 @@ async function runSqlQuery( if (opts?.countTotalRows) { return processRowCountResponse(response) } else if (Array.isArray(response)) { - return reverseUserColumnMapping(response) + return reverseUserColumnMapping(response, json.meta.table) } return response } @@ -368,17 +378,6 @@ export async function search( }) ) - // make sure relationships have columns reversed correctly - for (let columnName of Object.keys(table.schema)) { - if (table.schema[columnName].type !== FieldType.LINK) { - continue - } - // process the relationships (JSON generated by SQS) - for (let row of processed) { - row[columnName] = reverseUserColumnMapping(row[columnName]) - } - } - // check for pagination final row let nextRow: boolean = false if (paginate && params.limit && rows.length > params.limit) { From d1b12b8d4a5c7a0c74c589597bb9aa73e0f7d163 Mon Sep 17 00:00:00 2001 From: mike12345567 Date: Tue, 10 Sep 2024 15:52:41 +0100 Subject: [PATCH 71/85] Linting. --- packages/server/src/sdk/app/rows/search/internal/sqs.ts | 1 - 1 file changed, 1 deletion(-) diff --git a/packages/server/src/sdk/app/rows/search/internal/sqs.ts b/packages/server/src/sdk/app/rows/search/internal/sqs.ts index cf540a244b..560a39e70a 100644 --- a/packages/server/src/sdk/app/rows/search/internal/sqs.ts +++ b/packages/server/src/sdk/app/rows/search/internal/sqs.ts @@ -18,7 +18,6 @@ import { } from "@budibase/types" import { buildInternalRelationships, - fixJsonTypes, sqlOutputProcessing, } from "../../../../../api/controllers/row/utils" import sdk from "../../../../index" From 1ee8a12e290505ad5664823d388b6783005b04a8 Mon Sep 17 00:00:00 2001 From: Martin McKeaveney Date: Tue, 10 Sep 2024 16:01:00 +0100 Subject: [PATCH 72/85] use real data instead of mocks --- .../controllers/global/tests/configs.spec.ts | 159 +++++++----------- packages/worker/src/tests/api/configs.ts | 9 + .../worker/src/tests/structures/configs.ts | 19 ++- 3 files changed, 86 insertions(+), 101 deletions(-) diff --git a/packages/worker/src/api/controllers/global/tests/configs.spec.ts b/packages/worker/src/api/controllers/global/tests/configs.spec.ts index 419c12ccb0..1a6e6b7075 100644 --- a/packages/worker/src/api/controllers/global/tests/configs.spec.ts +++ b/packages/worker/src/api/controllers/global/tests/configs.spec.ts @@ -1,46 +1,36 @@ -import { configs } from "@budibase/backend-core" -import { UserCtx } from "@budibase/types" import * as pro from "@budibase/pro" -import { find, verifyAIConfig } from "../configs" - -jest.mock("@budibase/backend-core", () => ({ - ...jest.requireActual("@budibase/backend-core"), - configs: { - getConfig: jest.fn(), - save: jest.fn(), - }, -})) +import { verifyAIConfig } from "../configs" +import { TestConfiguration, structures } from "../../../../tests" describe("Global configs controller", () => { + const config = new TestConfiguration() + + beforeAll(async () => { + await config.beforeAll() + }) + + afterAll(async () => { + await config.afterAll() + }) + afterEach(() => { jest.resetAllMocks() }) it("Should strip secrets when pulling AI config", async () => { - configs.getConfig.mockResolvedValue({ - config: { - ai: { - apiKey: "abc123APIKey", - baseUrl: "https://api.example.com", - }, - }, - }) - const ctx = { - params: { - type: "ai", - }, - throw: jest.fn(), - } as UserCtx - - await find(ctx) - - expect(ctx.body).toEqual({ - config: { - ai: { - apiKey: "--secret-value--", - baseUrl: "https://api.example.com", - }, - }, + const data = structures.configs.ai() + await config.api.configs.saveConfig(data) + const response = await config.api.configs.getAIConfig() + expect(response.body.config).toEqual({ + ai: { + active: true, + apiKey: "--secret-value--", + baseUrl: "https://api.example.com", + defaultModel: "gpt4", + isDefault: false, + name: "Test", + provider: "OpenAI" + } }) }) @@ -48,36 +38,25 @@ describe("Global configs controller", () => { jest .spyOn(pro.features, "isBudibaseAIEnabled") .mockImplementation(() => true) - configs.getConfig.mockResolvedValue({ - config: { - ai: { - apiKey: "abc123APIKey", - baseUrl: "https://api.example.com", - }, - }, - }) - const ctx = { - params: { - type: "ai", - }, - throw: jest.fn(), - } as UserCtx + const data = structures.configs.ai() + await config.api.configs.saveConfig(data) + const response = await config.api.configs.getAIConfig() - await find(ctx) - - expect(ctx.body).toEqual({ - config: { - budibase_ai: { - provider: "OpenAI", - active: true, - isDefault: true, - defaultModel: undefined, - name: "Budibase AI", - }, - ai: { - apiKey: "--secret-value--", - baseUrl: "https://api.example.com", - }, + expect(response.body.config).toEqual({ + budibase_ai: { + provider: "OpenAI", + active: true, + isDefault: true, + name: "Budibase AI", + }, + ai: { + active: true, + apiKey: "--secret-value--", + baseUrl: "https://api.example.com", + defaultModel: "gpt4", + isDefault: false, + name: "Test", + provider: "OpenAI" }, }) }) @@ -86,49 +65,29 @@ describe("Global configs controller", () => { jest .spyOn(pro.features, "isBudibaseAIEnabled") .mockImplementation(() => false) - configs.getConfig.mockResolvedValue({ - config: { - ai: { - apiKey: "abc123APIKey", - baseUrl: "https://api.example.com", - }, - }, - }) - const ctx = { - params: { - type: "ai", - }, - throw: jest.fn(), - } as UserCtx + const data = structures.configs.ai() + await config.api.configs.saveConfig(data) + const response = await config.api.configs.getAIConfig() - await find(ctx) - - expect(ctx.body).toEqual({ - config: { - ai: { - apiKey: "--secret-value--", - baseUrl: "https://api.example.com", - }, + expect(response.body.config).toEqual({ + ai: { + active: true, + apiKey: "--secret-value--", + baseUrl: "https://api.example.com", + defaultModel: "gpt4", + isDefault: false, + name: "Test", + provider: "OpenAI" }, }) }) it("Should not update existing secrets when updating an existing AI Config", async () => { - const newConfig = { - type: "ai", - config: { - aiconfig: { - provider: "OpenAI", - isDefault: true, - name: "MyConfig", - active: true, - defaultModel: "gpt4", - apiKey: "--secret-value--", - }, - }, - } + const data = structures.configs.ai() + await config.api.configs.saveConfig(data) + const existingConfig = await config.api.configs.getAIConfig() - const existingConfig = { + const newConfig = { type: "ai", config: { aiconfig: { diff --git a/packages/worker/src/tests/api/configs.ts b/packages/worker/src/tests/api/configs.ts index 86fe0830ca..c64419b490 100644 --- a/packages/worker/src/tests/api/configs.ts +++ b/packages/worker/src/tests/api/configs.ts @@ -22,6 +22,15 @@ export class ConfigAPI extends TestAPI { .expect("Content-Type", /json/) } + getAIConfig = () => { + return this.request + .get(`/api/global/configs/ai`) + .set(this.config.defaultHeaders()) + .expect(200) + .expect("Content-Type", /json/) + } + + saveConfig = (data: any) => { return this.request .post(`/api/global/configs`) diff --git a/packages/worker/src/tests/structures/configs.ts b/packages/worker/src/tests/structures/configs.ts index bac94a4154..e1fd835d41 100644 --- a/packages/worker/src/tests/structures/configs.ts +++ b/packages/worker/src/tests/structures/configs.ts @@ -4,7 +4,7 @@ import { ConfigType, SMTPConfig, GoogleConfig, - OIDCConfig, + OIDCConfig, AIConfig, } from "@budibase/types" export function oidc(conf?: any): OIDCConfig { @@ -81,3 +81,20 @@ export function settings(conf?: any): SettingsConfig { }, } } + +export function ai(): AIConfig { + return { + type: ConfigType.AI, + config: { + ai: { + provider: "OpenAI", + isDefault: false, + name: "Test", + active: true, + defaultModel: "gpt4", + apiKey: "abc123APIKey", + baseUrl: "https://api.example.com", + }, + } + } +} From 86a6664c84445861b76477b90871cfaf3e199843 Mon Sep 17 00:00:00 2001 From: mike12345567 Date: Tue, 10 Sep 2024 16:06:53 +0100 Subject: [PATCH 73/85] Updating test case. --- .../src/integrations/tests/sqlAlias.spec.ts | 54 ++++++++++++------- 1 file changed, 36 insertions(+), 18 deletions(-) diff --git a/packages/server/src/integrations/tests/sqlAlias.spec.ts b/packages/server/src/integrations/tests/sqlAlias.spec.ts index 1ba37018dc..528e782543 100644 --- a/packages/server/src/integrations/tests/sqlAlias.spec.ts +++ b/packages/server/src/integrations/tests/sqlAlias.spec.ts @@ -32,7 +32,7 @@ function multiline(sql: string) { } describe("Captures of real examples", () => { - const baseLimit = 5000 + const relationshipLimit = 500 const primaryLimit = 100 function getJson(name: string): QueryJson { @@ -42,7 +42,9 @@ describe("Captures of real examples", () => { describe("create", () => { it("should create a row with relationships", () => { const queryJson = getJson("createWithRelationships.json") - let query = new Sql(SqlClient.POSTGRES, baseLimit)._query(queryJson) + let query = new Sql(SqlClient.POSTGRES, relationshipLimit)._query( + queryJson + ) expect(query).toEqual({ bindings: ["A Street", 34, "London", "A", "B", "designer", 1990], sql: multiline(`insert into "persons" ("address", "age", "city", "firstname", "lastname", "type", "year") @@ -54,9 +56,11 @@ describe("Captures of real examples", () => { describe("read", () => { it("should handle basic retrieval with relationships", () => { const queryJson = getJson("basicFetchWithRelationships.json") - let query = new Sql(SqlClient.POSTGRES, baseLimit)._query(queryJson) + let query = new Sql(SqlClient.POSTGRES, relationshipLimit)._query( + queryJson + ) expect(query).toEqual({ - bindings: [baseLimit, baseLimit, primaryLimit], + bindings: [relationshipLimit, relationshipLimit, primaryLimit], sql: expect.stringContaining( multiline( `select json_agg(json_build_object('executorid',"b"."executorid",'taskname',"b"."taskname",'taskid',"b"."taskid",'completed',"b"."completed",'qaid',"b"."qaid",'executorid',"b"."executorid",'taskname',"b"."taskname",'taskid',"b"."taskid",'completed',"b"."completed",'qaid',"b"."qaid")` @@ -67,9 +71,11 @@ describe("Captures of real examples", () => { it("should handle filtering by relationship", () => { const queryJson = getJson("filterByRelationship.json") - let query = new Sql(SqlClient.POSTGRES, baseLimit)._query(queryJson) + let query = new Sql(SqlClient.POSTGRES, relationshipLimit)._query( + queryJson + ) expect(query).toEqual({ - bindings: [baseLimit, "assembling", primaryLimit], + bindings: [relationshipLimit, "assembling", primaryLimit], sql: expect.stringContaining( multiline( `where exists (select 1 from "tasks" as "b" inner join "products_tasks" as "c" on "b"."taskid" = "c"."taskid" @@ -81,9 +87,11 @@ describe("Captures of real examples", () => { it("should handle fetching many to many relationships", () => { const queryJson = getJson("fetchManyToMany.json") - let query = new Sql(SqlClient.POSTGRES, baseLimit)._query(queryJson) + let query = new Sql(SqlClient.POSTGRES, relationshipLimit)._query( + queryJson + ) expect(query).toEqual({ - bindings: [baseLimit, primaryLimit], + bindings: [relationshipLimit, primaryLimit], sql: expect.stringContaining( multiline( `select json_agg(json_build_object('executorid',"b"."executorid",'taskname',"b"."taskname",'taskid',"b"."taskid",'completed',"b"."completed",'qaid',"b"."qaid")) @@ -97,9 +105,11 @@ describe("Captures of real examples", () => { it("should handle enrichment of rows", () => { const queryJson = getJson("enrichRelationship.json") const filters = queryJson.filters?.oneOf?.taskid as number[] - let query = new Sql(SqlClient.POSTGRES, baseLimit)._query(queryJson) + let query = new Sql(SqlClient.POSTGRES, relationshipLimit)._query( + queryJson + ) expect(query).toEqual({ - bindings: [baseLimit, ...filters, baseLimit], + bindings: [relationshipLimit, ...filters, relationshipLimit], sql: multiline( `select "a".*, (select json_agg(json_build_object('productname',"b"."productname",'productid',"b"."productid")) from (select "b".* from "products" as "b" inner join "products_tasks" as "c" on "b"."productid" = "c"."productid" @@ -111,7 +121,9 @@ describe("Captures of real examples", () => { it("should manage query with many relationship filters", () => { const queryJson = getJson("manyRelationshipFilters.json") - let query = new Sql(SqlClient.POSTGRES, baseLimit)._query(queryJson) + let query = new Sql(SqlClient.POSTGRES, relationshipLimit)._query( + queryJson + ) const filters = queryJson.filters const notEqualsValue = Object.values(filters?.notEqual!)[0] const rangeValue: { high?: string | number; low?: string | number } = @@ -120,9 +132,9 @@ describe("Captures of real examples", () => { expect(query).toEqual({ bindings: [ - baseLimit, - baseLimit, - baseLimit, + relationshipLimit, + relationshipLimit, + relationshipLimit, rangeValue.low, rangeValue.high, equalValue, @@ -141,7 +153,9 @@ describe("Captures of real examples", () => { describe("update", () => { it("should handle performing a simple update", () => { const queryJson = getJson("updateSimple.json") - let query = new Sql(SqlClient.POSTGRES, baseLimit)._query(queryJson) + let query = new Sql(SqlClient.POSTGRES, relationshipLimit)._query( + queryJson + ) expect(query).toEqual({ bindings: [1990, "C", "A Street", 34, "designer", "London", "B", 5], sql: multiline( @@ -153,7 +167,9 @@ describe("Captures of real examples", () => { it("should handle performing an update of relationships", () => { const queryJson = getJson("updateRelationship.json") - let query = new Sql(SqlClient.POSTGRES, baseLimit)._query(queryJson) + let query = new Sql(SqlClient.POSTGRES, relationshipLimit)._query( + queryJson + ) expect(query).toEqual({ bindings: [1990, "C", "A Street", 34, "designer", "London", "B", 5], sql: multiline(`update "persons" as "a" set "year" = $1, "firstname" = $2, "address" = $3, "age" = $4, @@ -165,7 +181,9 @@ describe("Captures of real examples", () => { describe("delete", () => { it("should handle deleting with relationships", () => { const queryJson = getJson("deleteSimple.json") - let query = new Sql(SqlClient.POSTGRES, baseLimit)._query(queryJson) + let query = new Sql(SqlClient.POSTGRES, relationshipLimit)._query( + queryJson + ) expect(query).toEqual({ bindings: ["ddd", ""], sql: multiline(`delete from "compositetable" as "a" @@ -178,7 +196,7 @@ describe("Captures of real examples", () => { describe("returning (everything bar Postgres)", () => { it("should be able to handle row returning", () => { const queryJson = getJson("createSimple.json") - const SQL = new Sql(SqlClient.MS_SQL, baseLimit) + const SQL = new Sql(SqlClient.MS_SQL, relationshipLimit) let query = SQL._query(queryJson, { disableReturning: true }) expect(query).toEqual({ sql: "insert into [people] ([age], [name]) values (@p0, @p1)", From c6907eb09e325b167dd83db93cf7fdb12ea1602d Mon Sep 17 00:00:00 2001 From: Martin McKeaveney Date: Tue, 10 Sep 2024 16:41:44 +0100 Subject: [PATCH 74/85] fix types --- packages/builder/src/stores/portal/menu.js | 4 ++- packages/types/src/documents/global/config.ts | 2 +- .../src/api/controllers/global/configs.ts | 9 +++--- .../controllers/global/tests/configs.spec.ts | 31 +++++++++---------- packages/worker/src/tests/api/configs.ts | 1 - .../worker/src/tests/structures/configs.ts | 5 +-- 6 files changed, 26 insertions(+), 26 deletions(-) diff --git a/packages/builder/src/stores/portal/menu.js b/packages/builder/src/stores/portal/menu.js index 19b190fd24..a1b5450ea4 100644 --- a/packages/builder/src/stores/portal/menu.js +++ b/packages/builder/src/stores/portal/menu.js @@ -84,7 +84,9 @@ export const menu = derived([admin, auth], ([$admin, $auth]) => { menu.push({ title: "Settings", href: "/builder/portal/settings", - subPages: [...settingsSubPages].sort((a, b) => a.title.localeCompare(b.title)), + subPages: [...settingsSubPages].sort((a, b) => + a.title.localeCompare(b.title) + ), }) } diff --git a/packages/types/src/documents/global/config.ts b/packages/types/src/documents/global/config.ts index 6e9c29af97..8d64b49ee9 100644 --- a/packages/types/src/documents/global/config.ts +++ b/packages/types/src/documents/global/config.ts @@ -121,7 +121,7 @@ export interface AIInnerConfig { active: boolean baseUrl?: string apiKey?: string - defaultModel: string + defaultModel?: string } } diff --git a/packages/worker/src/api/controllers/global/configs.ts b/packages/worker/src/api/controllers/global/configs.ts index d51ae65c51..0760a6c692 100644 --- a/packages/worker/src/api/controllers/global/configs.ts +++ b/packages/worker/src/api/controllers/global/configs.ts @@ -32,6 +32,7 @@ import { AIConfig, PASSWORD_REPLACEMENT, isAIConfig, + AIInnerConfig, } from "@budibase/types" import * as pro from "@budibase/pro" @@ -210,13 +211,13 @@ async function verifyOIDCConfig(config: OIDCConfigs) { } export async function verifyAIConfig( - config: AIConfig, + configToSave: AIInnerConfig, existingConfig: AIConfig ) { // ensure that the redacted API keys are not overwritten in the DB for (let uuid in existingConfig.config) { - if (config[uuid]?.apiKey === PASSWORD_REPLACEMENT) { - config[uuid].apiKey = existingConfig.config[uuid].apiKey + if (configToSave[uuid]?.apiKey === PASSWORD_REPLACEMENT) { + configToSave[uuid].apiKey = existingConfig.config[uuid].apiKey } } } @@ -351,7 +352,7 @@ async function enrichAIConfig(aiConfig: AIConfig) { provider: "OpenAI", active: true, isDefault: !defaultConfigExists, - defaultModel: env.BUDIBASE_AI_DEFAULT_MODEL, + defaultModel: env.BUDIBASE_AI_DEFAULT_MODEL || "", name: "Budibase AI", } } diff --git a/packages/worker/src/api/controllers/global/tests/configs.spec.ts b/packages/worker/src/api/controllers/global/tests/configs.spec.ts index 1a6e6b7075..06d1ad0b2f 100644 --- a/packages/worker/src/api/controllers/global/tests/configs.spec.ts +++ b/packages/worker/src/api/controllers/global/tests/configs.spec.ts @@ -1,6 +1,7 @@ import * as pro from "@budibase/pro" import { verifyAIConfig } from "../configs" import { TestConfiguration, structures } from "../../../../tests" +import { AIInnerConfig } from "@budibase/types" describe("Global configs controller", () => { const config = new TestConfiguration() @@ -29,8 +30,8 @@ describe("Global configs controller", () => { defaultModel: "gpt4", isDefault: false, name: "Test", - provider: "OpenAI" - } + provider: "OpenAI", + }, }) }) @@ -56,7 +57,7 @@ describe("Global configs controller", () => { defaultModel: "gpt4", isDefault: false, name: "Test", - provider: "OpenAI" + provider: "OpenAI", }, }) }) @@ -77,7 +78,7 @@ describe("Global configs controller", () => { defaultModel: "gpt4", isDefault: false, name: "Test", - provider: "OpenAI" + provider: "OpenAI", }, }) }) @@ -87,22 +88,18 @@ describe("Global configs controller", () => { await config.api.configs.saveConfig(data) const existingConfig = await config.api.configs.getAIConfig() - const newConfig = { - type: "ai", - config: { - aiconfig: { - provider: "OpenAI", - isDefault: true, - name: "MyConfig", - active: true, - defaultModel: "gpt4", - apiKey: "myapikey", - }, + const newConfig: AIInnerConfig = { + aiconfig: { + provider: "OpenAI", + isDefault: true, + name: "MyConfig", + active: true, + defaultModel: "gpt4", }, } - await verifyAIConfig(newConfig, existingConfig) + await verifyAIConfig(newConfig, existingConfig.body) // should be unchanged - expect(newConfig.config.aiconfig.apiKey).toEqual("myapikey") + expect(newConfig.aiconfig.apiKey).toEqual("myapikey") }) }) diff --git a/packages/worker/src/tests/api/configs.ts b/packages/worker/src/tests/api/configs.ts index c64419b490..e0c1c806af 100644 --- a/packages/worker/src/tests/api/configs.ts +++ b/packages/worker/src/tests/api/configs.ts @@ -30,7 +30,6 @@ export class ConfigAPI extends TestAPI { .expect("Content-Type", /json/) } - saveConfig = (data: any) => { return this.request .post(`/api/global/configs`) diff --git a/packages/worker/src/tests/structures/configs.ts b/packages/worker/src/tests/structures/configs.ts index e1fd835d41..9257e813cb 100644 --- a/packages/worker/src/tests/structures/configs.ts +++ b/packages/worker/src/tests/structures/configs.ts @@ -4,7 +4,8 @@ import { ConfigType, SMTPConfig, GoogleConfig, - OIDCConfig, AIConfig, + OIDCConfig, + AIConfig, } from "@budibase/types" export function oidc(conf?: any): OIDCConfig { @@ -95,6 +96,6 @@ export function ai(): AIConfig { apiKey: "abc123APIKey", baseUrl: "https://api.example.com", }, - } + }, } } From 87adb99073e31cc4fdca7149cd07d9b7c74285e2 Mon Sep 17 00:00:00 2001 From: Martin McKeaveney Date: Tue, 10 Sep 2024 16:42:56 +0100 Subject: [PATCH 75/85] update pro ref --- packages/pro | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/pro b/packages/pro index 62586bc47b..ec1d2bda75 160000 --- a/packages/pro +++ b/packages/pro @@ -1 +1 @@ -Subproject commit 62586bc47bf2af507356b0e1e64df8b0ae9fad43 +Subproject commit ec1d2bda756f02c6b4efdee086e4c59b0c2a1b0c From 304b2a5c61853e4e71f0216a086d1344688408bb Mon Sep 17 00:00:00 2001 From: Martin McKeaveney Date: Tue, 10 Sep 2024 16:45:32 +0100 Subject: [PATCH 76/85] Update packages/worker/src/api/controllers/global/configs.ts Co-authored-by: Sam Rose --- packages/worker/src/api/controllers/global/configs.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/worker/src/api/controllers/global/configs.ts b/packages/worker/src/api/controllers/global/configs.ts index 0760a6c692..70b2279f6c 100644 --- a/packages/worker/src/api/controllers/global/configs.ts +++ b/packages/worker/src/api/controllers/global/configs.ts @@ -215,7 +215,7 @@ export async function verifyAIConfig( existingConfig: AIConfig ) { // ensure that the redacted API keys are not overwritten in the DB - for (let uuid in existingConfig.config) { + for (const uuid in existingConfig.config) { if (configToSave[uuid]?.apiKey === PASSWORD_REPLACEMENT) { configToSave[uuid].apiKey = existingConfig.config[uuid].apiKey } From 5c9f88de59c8d84aaf90be316d7a0446eaecd698 Mon Sep 17 00:00:00 2001 From: Martin McKeaveney Date: Tue, 10 Sep 2024 16:51:22 +0100 Subject: [PATCH 77/85] account portal ref --- packages/account-portal | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/account-portal b/packages/account-portal index ce60bf9231..4bef13d035 160000 --- a/packages/account-portal +++ b/packages/account-portal @@ -1 +1 @@ -Subproject commit ce60bf923115855a9cd0bb991297cccd308eae13 +Subproject commit 4bef13d035217b65c7f2107e1f8dfc04e7a56787 From dbfc6851af53f04139b2fb5b4b6c5a349ff2bf68 Mon Sep 17 00:00:00 2001 From: Sam Rose Date: Tue, 10 Sep 2024 16:58:33 +0100 Subject: [PATCH 78/85] More row reading tests. --- .../integrations/tests/googlesheets.spec.ts | 83 +++++++++++++++++++ .../integrations/tests/utils/googlesheets.ts | 48 +++++++++++ 2 files changed, 131 insertions(+) diff --git a/packages/server/src/integrations/tests/googlesheets.spec.ts b/packages/server/src/integrations/tests/googlesheets.spec.ts index 4db85a80c2..9781b97972 100644 --- a/packages/server/src/integrations/tests/googlesheets.spec.ts +++ b/packages/server/src/integrations/tests/googlesheets.spec.ts @@ -127,6 +127,89 @@ describe("Google Sheets Integration", () => { }) }) + describe("read", () => { + let table: Table + beforeEach(async () => { + table = await config.api.table.save({ + name: "Test Table", + type: "table", + sourceId: datasource._id!, + sourceType: TableSourceType.EXTERNAL, + schema: { + name: { + name: "name", + type: FieldType.STRING, + constraints: { + type: "string", + }, + }, + description: { + name: "description", + type: FieldType.STRING, + constraints: { + type: "string", + }, + }, + }, + }) + + await config.api.row.bulkImport(table._id!, { + rows: [ + { + name: "Test Contact 1", + description: "original description 1", + }, + { + name: "Test Contact 2", + description: "original description 2", + }, + ], + }) + }) + + it("can read table details", async () => { + const response = await config.api.table.get(table._id!) + expect(response.name).toEqual("Test Table") + expect(response.schema).toEqual({ + name: { + name: "name", + type: FieldType.STRING, + constraints: { + type: "string", + }, + }, + description: { + name: "description", + type: FieldType.STRING, + constraints: { + type: "string", + }, + }, + }) + }) + + it("can read table rows", async () => { + const rows = await config.api.row.fetch(table._id!) + expect(rows.length).toEqual(2) + expect(rows[0].name).toEqual("Test Contact 1") + expect(rows[0].description).toEqual("original description 1") + expect(rows[0]._id).toEqual("%5B2%5D") + expect(rows[1].name).toEqual("Test Contact 2") + expect(rows[1].description).toEqual("original description 2") + expect(rows[1]._id).toEqual("%5B3%5D") + }) + + it("can get a specific row", async () => { + const row1 = await config.api.row.get(table._id!, "2") + expect(row1.name).toEqual("Test Contact 1") + expect(row1.description).toEqual("original description 1") + + const row2 = await config.api.row.get(table._id!, "3") + expect(row2.name).toEqual("Test Contact 2") + expect(row2.description).toEqual("original description 2") + }) + }) + describe("update", () => { let table: Table beforeEach(async () => { diff --git a/packages/server/src/integrations/tests/utils/googlesheets.ts b/packages/server/src/integrations/tests/utils/googlesheets.ts index 0f92fea6cd..c58066bee5 100644 --- a/packages/server/src/integrations/tests/utils/googlesheets.ts +++ b/packages/server/src/integrations/tests/utils/googlesheets.ts @@ -534,9 +534,57 @@ export class GoogleSheetsMock { } values.push(this.cellValue(cell)) } + valueRange.values.push(values) } + return this.trimValueRange(valueRange) + } + + // When Google Sheets returns a value range, it will trim the data down to the + // smallest possible size. It does all of the following: + // + // 1. Converts cells in non-empty rows up to the first value to empty strings. + // 2. Removes all cells after the last non-empty cell in a row. + // 3. Removes all rows after the last non-empty row. + // 4. Rows that are before the first non-empty row that are empty are replaced with []. + // + // We replicate this behaviour here. + private trimValueRange(valueRange: ValueRange): ValueRange { + for (const row of valueRange.values) { + if (row.every(v => v == null)) { + row.splice(0, row.length) + continue + } + + for (let i = row.length - 1; i >= 0; i--) { + const cell = row[i] + if (cell == null) { + row.pop() + } else { + break + } + } + + for (let i = 0; i < row.length; i++) { + const cell = row[i] + if (cell == null) { + row[i] = "" + } else { + break + } + } + } + + for (let i = valueRange.values.length - 1; i >= 0; i--) { + const row = valueRange.values[i] + if (row.length === 0) { + valueRange.values.pop() + } else { + break + } + } + return valueRange } From 1582e3221faa8f7eac1d87e9eb9eb3e5a11622fa Mon Sep 17 00:00:00 2001 From: mike12345567 Date: Tue, 10 Sep 2024 17:04:59 +0100 Subject: [PATCH 79/85] Adding test case for getting related array column in a JS formula. --- .../src/api/routes/tests/search.spec.ts | 63 ++++++++++++++++++- 1 file changed, 62 insertions(+), 1 deletion(-) diff --git a/packages/server/src/api/routes/tests/search.spec.ts b/packages/server/src/api/routes/tests/search.spec.ts index f5c5ade2f8..edac4c6662 100644 --- a/packages/server/src/api/routes/tests/search.spec.ts +++ b/packages/server/src/api/routes/tests/search.spec.ts @@ -9,10 +9,10 @@ import { db as dbCore, MAX_VALID_DATE, MIN_VALID_DATE, + setEnv as setCoreEnv, SQLITE_DESIGN_DOC_ID, utils, withEnv as withCoreEnv, - setEnv as setCoreEnv, } from "@budibase/backend-core" import * as setup from "./utilities" @@ -1937,6 +1937,67 @@ describe.each([ }) }) + isSql && + describe("related formulas", () => { + beforeAll(async () => { + const arrayTable = await createTable( + { + name: { name: "name", type: FieldType.STRING }, + array: { + name: "array", + type: FieldType.ARRAY, + constraints: { + type: JsonFieldSubType.ARRAY, + inclusion: ["option 1", "option 2"], + }, + }, + }, + "array" + ) + table = await createTable( + { + relationship: { + type: FieldType.LINK, + relationshipType: RelationshipType.ONE_TO_MANY, + name: "relationship", + fieldName: "relate", + tableId: arrayTable._id!, + constraints: { + type: "array", + }, + }, + formula: { + type: FieldType.FORMULA, + name: "formula", + formula: encodeJSBinding( + `let array = [];$("relationship").forEach(rel => array = array.concat(rel.array));return array.join(",")` + ), + }, + }, + "main" + ) + const arrayRows = await Promise.all([ + config.api.row.save(arrayTable._id!, { + name: "foo", + array: ["option 1"], + }), + config.api.row.save(arrayTable._id!, { + name: "bar", + array: ["option 2"], + }), + ]) + await Promise.all([ + config.api.row.save(table._id!, { + relationship: [arrayRows[0]._id, arrayRows[1]._id], + }), + ]) + }) + + it("formula is correct with relationship arrays", async () => { + await expectQuery({}).toContain([{ formula: "option 1,option 2" }]) + }) + }) + describe("user", () => { let user1: User let user2: User From 9a61ec5950ecb63e89f586d59c73be14408a1d6b Mon Sep 17 00:00:00 2001 From: mike12345567 Date: Tue, 10 Sep 2024 17:07:31 +0100 Subject: [PATCH 80/85] More incorrect limits. --- packages/server/src/integrations/tests/sql.spec.ts | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/packages/server/src/integrations/tests/sql.spec.ts b/packages/server/src/integrations/tests/sql.spec.ts index c434ec42cb..f2bcc0bde0 100644 --- a/packages/server/src/integrations/tests/sql.spec.ts +++ b/packages/server/src/integrations/tests/sql.spec.ts @@ -149,6 +149,7 @@ function generateManyRelationshipJson(config: { schema?: string } = {}) { } describe("SQL query builder", () => { + const relationshipLimit = 500 const limit = 500 const client = SqlClient.POSTGRES let sql: any @@ -160,7 +161,7 @@ describe("SQL query builder", () => { it("should add the schema to the LEFT JOIN", () => { const query = sql._query(generateRelationshipJson({ schema: "production" })) expect(query).toEqual({ - bindings: [5000, limit], + bindings: [relationshipLimit, limit], sql: `select "brands".*, (select json_agg(json_build_object('product_id',"products"."product_id",'product_name',"products"."product_name",'brand_id',"products"."brand_id")) from (select "products".* from "production"."products" as "products" where "products"."brand_id" = "brands"."brand_id" order by "products"."brand_id" asc limit $1) as "products") as "products" from "production"."brands" order by "test"."id" asc limit $2`, }) }) @@ -168,7 +169,7 @@ describe("SQL query builder", () => { it("should handle if the schema is not present when doing a LEFT JOIN", () => { const query = sql._query(generateRelationshipJson()) expect(query).toEqual({ - bindings: [5000, limit], + bindings: [relationshipLimit, limit], sql: `select "brands".*, (select json_agg(json_build_object('product_id',"products"."product_id",'product_name',"products"."product_name",'brand_id',"products"."brand_id")) from (select "products".* from "products" as "products" where "products"."brand_id" = "brands"."brand_id" order by "products"."brand_id" asc limit $1) as "products") as "products" from "brands" order by "test"."id" asc limit $2`, }) }) @@ -178,7 +179,7 @@ describe("SQL query builder", () => { generateManyRelationshipJson({ schema: "production" }) ) expect(query).toEqual({ - bindings: [5000, limit], + bindings: [relationshipLimit, limit], sql: `select "stores".*, (select json_agg(json_build_object('product_id',"products"."product_id",'product_name',"products"."product_name")) from (select "products".* from "production"."products" as "products" inner join "production"."stocks" as "stocks" on "products"."product_id" = "stocks"."product_id" where "stocks"."store_id" = "stores"."store_id" order by "products"."product_id" asc limit $1) as "products") as "products" from "production"."stores" order by "test"."id" asc limit $2`, }) }) From 595dd7ea6d31cd092f2adb7ccb0f54e3ffd86d3b Mon Sep 17 00:00:00 2001 From: mike12345567 Date: Tue, 10 Sep 2024 17:21:36 +0100 Subject: [PATCH 81/85] Fix for test case. --- packages/server/src/api/routes/tests/search.spec.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/packages/server/src/api/routes/tests/search.spec.ts b/packages/server/src/api/routes/tests/search.spec.ts index edac4c6662..26bb1582dd 100644 --- a/packages/server/src/api/routes/tests/search.spec.ts +++ b/packages/server/src/api/routes/tests/search.spec.ts @@ -1958,7 +1958,7 @@ describe.each([ { relationship: { type: FieldType.LINK, - relationshipType: RelationshipType.ONE_TO_MANY, + relationshipType: RelationshipType.MANY_TO_ONE, name: "relationship", fieldName: "relate", tableId: arrayTable._id!, @@ -1970,7 +1970,7 @@ describe.each([ type: FieldType.FORMULA, name: "formula", formula: encodeJSBinding( - `let array = [];$("relationship").forEach(rel => array = array.concat(rel.array));return array.join(",")` + `let array = [];$("relationship").forEach(rel => array = array.concat(rel.array));return array.sort().join(",")` ), }, }, From 38fc65c5419082dd106c8fab764f9f6a5de61842 Mon Sep 17 00:00:00 2001 From: Martin McKeaveney Date: Tue, 10 Sep 2024 22:25:47 +0100 Subject: [PATCH 82/85] fix tests --- .../src/api/controllers/global/tests/configs.spec.ts | 9 +++++---- packages/worker/src/tests/structures/configs.ts | 2 +- 2 files changed, 6 insertions(+), 5 deletions(-) diff --git a/packages/worker/src/api/controllers/global/tests/configs.spec.ts b/packages/worker/src/api/controllers/global/tests/configs.spec.ts index 06d1ad0b2f..ad9efc4398 100644 --- a/packages/worker/src/api/controllers/global/tests/configs.spec.ts +++ b/packages/worker/src/api/controllers/global/tests/configs.spec.ts @@ -49,6 +49,7 @@ describe("Global configs controller", () => { active: true, isDefault: true, name: "Budibase AI", + defaultModel: "", }, ai: { active: true, @@ -86,20 +87,20 @@ describe("Global configs controller", () => { it("Should not update existing secrets when updating an existing AI Config", async () => { const data = structures.configs.ai() await config.api.configs.saveConfig(data) - const existingConfig = await config.api.configs.getAIConfig() const newConfig: AIInnerConfig = { - aiconfig: { + ai: { provider: "OpenAI", isDefault: true, + apiKey: "--secret-value--", name: "MyConfig", active: true, defaultModel: "gpt4", }, } - await verifyAIConfig(newConfig, existingConfig.body) + await verifyAIConfig(newConfig, data) // should be unchanged - expect(newConfig.aiconfig.apiKey).toEqual("myapikey") + expect(newConfig.ai.apiKey).toEqual("myapikey") }) }) diff --git a/packages/worker/src/tests/structures/configs.ts b/packages/worker/src/tests/structures/configs.ts index 9257e813cb..5aa073408c 100644 --- a/packages/worker/src/tests/structures/configs.ts +++ b/packages/worker/src/tests/structures/configs.ts @@ -93,7 +93,7 @@ export function ai(): AIConfig { name: "Test", active: true, defaultModel: "gpt4", - apiKey: "abc123APIKey", + apiKey: "myapikey", baseUrl: "https://api.example.com", }, }, From a27be54b691336f7edcd128fc2b2e7466f2035ef Mon Sep 17 00:00:00 2001 From: Martin McKeaveney Date: Tue, 10 Sep 2024 22:38:45 +0100 Subject: [PATCH 83/85] make mocks async --- .../worker/src/api/controllers/global/tests/configs.spec.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/packages/worker/src/api/controllers/global/tests/configs.spec.ts b/packages/worker/src/api/controllers/global/tests/configs.spec.ts index ad9efc4398..3ff6a5298c 100644 --- a/packages/worker/src/api/controllers/global/tests/configs.spec.ts +++ b/packages/worker/src/api/controllers/global/tests/configs.spec.ts @@ -38,7 +38,7 @@ describe("Global configs controller", () => { it("Should return the default BB AI config when the feature is turned on", async () => { jest .spyOn(pro.features, "isBudibaseAIEnabled") - .mockImplementation(() => true) + .mockImplementation(() => Promise.resolve(true)) const data = structures.configs.ai() await config.api.configs.saveConfig(data) const response = await config.api.configs.getAIConfig() @@ -66,7 +66,7 @@ describe("Global configs controller", () => { it("Should not not return the default Budibase AI config when on self host", async () => { jest .spyOn(pro.features, "isBudibaseAIEnabled") - .mockImplementation(() => false) + .mockImplementation(() => Promise.resolve(false)) const data = structures.configs.ai() await config.api.configs.saveConfig(data) const response = await config.api.configs.getAIConfig() From a5ff0d5ddc9c597bc8bb32fbe03c2c50cd799666 Mon Sep 17 00:00:00 2001 From: Martin McKeaveney Date: Tue, 10 Sep 2024 22:55:14 +0100 Subject: [PATCH 84/85] update acct portal ref --- packages/account-portal | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/account-portal b/packages/account-portal index 4bef13d035..c587ebc17a 160000 --- a/packages/account-portal +++ b/packages/account-portal @@ -1 +1 @@ -Subproject commit 4bef13d035217b65c7f2107e1f8dfc04e7a56787 +Subproject commit c587ebc17adf778bb02a77732cc0b2cc63399a93 From fcc03814744fcb55ac261de41f29697309f40f38 Mon Sep 17 00:00:00 2001 From: Martin McKeaveney Date: Wed, 11 Sep 2024 09:42:29 +0100 Subject: [PATCH 85/85] update account portal ref --- packages/account-portal | 2 +- packages/types/src/sdk/licensing/feature.ts | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/packages/account-portal b/packages/account-portal index c587ebc17a..048c37ecd9 160000 --- a/packages/account-portal +++ b/packages/account-portal @@ -1 +1 @@ -Subproject commit c587ebc17adf778bb02a77732cc0b2cc63399a93 +Subproject commit 048c37ecd921340614bf61a76a774aaa46176569 diff --git a/packages/types/src/sdk/licensing/feature.ts b/packages/types/src/sdk/licensing/feature.ts index cbcb16f7b6..9d09f1d14f 100644 --- a/packages/types/src/sdk/licensing/feature.ts +++ b/packages/types/src/sdk/licensing/feature.ts @@ -16,7 +16,7 @@ export enum Feature { VIEW_PERMISSIONS = "viewPermissions", VIEW_READONLY_COLUMNS = "viewReadonlyColumns", BUDIBASE_AI = "budibaseAI", - AI_CUSTOM_CONFIGS = "customAIConfigs", + AI_CUSTOM_CONFIGS = "aiCustomConfigs", } export type PlanFeatures = { [key in PlanType]: Feature[] | undefined }