diff --git a/packages/backend-core/src/sql/sql.ts b/packages/backend-core/src/sql/sql.ts index 34b950bf2c..615753efc3 100644 --- a/packages/backend-core/src/sql/sql.ts +++ b/packages/backend-core/src/sql/sql.ts @@ -184,7 +184,11 @@ class InternalBuilder { query: Knex.QueryBuilder, filters: SearchFilters | undefined, table: Table, - opts: { aliases?: Record; relationship?: boolean } + opts: { + aliases?: Record + relationship?: boolean + columnPrefix?: string + } ): Knex.QueryBuilder { if (!filters) { return query @@ -192,7 +196,10 @@ class InternalBuilder { filters = parseFilters(filters) // if all or specified in filters, then everything is an or const allOr = filters.allOr - const sqlStatements = new SqlStatements(this.client, table, { allOr }) + const sqlStatements = new SqlStatements(this.client, table, { + allOr, + columnPrefix: opts.columnPrefix, + }) const tableName = this.client === SqlClient.SQL_LITE ? table._id! : table.name @@ -663,6 +670,7 @@ class InternalBuilder { } // add filters to the query (where) query = this.addFilters(query, filters, json.meta.table, { + columnPrefix: json.meta.columnPrefix, aliases: tableAliases, }) @@ -698,6 +706,7 @@ class InternalBuilder { } return this.addFilters(query, filters, json.meta.table, { + columnPrefix: json.meta.columnPrefix, relationship: true, aliases: tableAliases, }) @@ -708,6 +717,7 @@ class InternalBuilder { let query = this.knexWithAlias(knex, endpoint, tableAliases) const parsedBody = parseBody(body) query = this.addFilters(query, filters, json.meta.table, { + columnPrefix: json.meta.columnPrefix, aliases: tableAliases, }) // mysql can't use returning @@ -722,6 +732,7 @@ class InternalBuilder { const { endpoint, filters, tableAliases } = json let query = this.knexWithAlias(knex, endpoint, tableAliases) query = this.addFilters(query, filters, json.meta.table, { + columnPrefix: json.meta.columnPrefix, aliases: tableAliases, }) // mysql can't use returning diff --git a/packages/backend-core/src/sql/sqlStatements.ts b/packages/backend-core/src/sql/sqlStatements.ts index a80defd8b8..311f7c7d49 100644 --- a/packages/backend-core/src/sql/sqlStatements.ts +++ b/packages/backend-core/src/sql/sqlStatements.ts @@ -5,19 +5,27 @@ export class SqlStatements { client: string table: Table allOr: boolean | undefined + columnPrefix: string | undefined + constructor( client: string, table: Table, - { allOr }: { allOr?: boolean } = {} + { allOr, columnPrefix }: { allOr?: boolean; columnPrefix?: string } = {} ) { this.client = client this.table = table this.allOr = allOr + this.columnPrefix = columnPrefix } getField(key: string): FieldSchema | undefined { const fieldName = key.split(".")[1] - return this.table.schema[fieldName] + let found = this.table.schema[fieldName] + if (!found && this.columnPrefix) { + const prefixRemovedFieldName = fieldName.replace(this.columnPrefix, "") + found = this.table.schema[prefixRemovedFieldName] + } + return found } between( diff --git a/packages/server/src/api/routes/tests/search.spec.ts b/packages/server/src/api/routes/tests/search.spec.ts index 5b98347953..2d01ec0f6d 100644 --- a/packages/server/src/api/routes/tests/search.spec.ts +++ b/packages/server/src/api/routes/tests/search.spec.ts @@ -2119,4 +2119,29 @@ describe.each([ }).toNotHaveProperty(["totalRows"]) }) }) + + describe.each(["data_name_test", "name_data_test", "name_test_data_"])( + "special (%s) case", + column => { + beforeAll(async () => { + table = await createTable({ + [column]: { + name: column, + type: FieldType.STRING, + }, + }) + await createRows([{ [column]: "a" }, { [column]: "b" }]) + }) + + it("should be able to query a column with data_ in it", async () => { + await expectSearch({ + query: { + equal: { + [`1:${column}`]: "a", + }, + }, + }).toContainExactly([{ [column]: "a" }]) + }) + } + ) }) diff --git a/packages/server/src/appMigrations/migrations/tests/20240604153647_initial_sqs.spec.ts b/packages/server/src/appMigrations/migrations/tests/20240604153647_initial_sqs.spec.ts index 86e50a5812..572e694855 100644 --- a/packages/server/src/appMigrations/migrations/tests/20240604153647_initial_sqs.spec.ts +++ b/packages/server/src/appMigrations/migrations/tests/20240604153647_initial_sqs.spec.ts @@ -15,6 +15,7 @@ import { import { processMigrations } from "../../migrationsProcessor" import migration from "../20240604153647_initial_sqs" import { AppMigration } from "src/appMigrations" +import sdk from "../../../sdk" const MIGRATIONS: AppMigration[] = [ { @@ -27,6 +28,8 @@ const MIGRATIONS: AppMigration[] = [ const config = setup.getConfig() let tableId: string +const prefix = sdk.tables.sqs.mapToUserColumn + function oldLinkDocInfo() { const tableId1 = `${DocumentType.TABLE}_a`, tableId2 = `${DocumentType.TABLE}_b` @@ -102,8 +105,14 @@ describe("SQS migration", () => { expect(designDoc.sql.tables).toBeDefined() const mainTableDef = designDoc.sql.tables[tableId] expect(mainTableDef).toBeDefined() - expect(mainTableDef.fields.name).toEqual(SQLiteType.TEXT) - expect(mainTableDef.fields.description).toEqual(SQLiteType.TEXT) + expect(mainTableDef.fields[prefix("name")]).toEqual({ + field: "name", + type: SQLiteType.TEXT, + }) + expect(mainTableDef.fields[prefix("description")]).toEqual({ + field: "description", + type: SQLiteType.TEXT, + }) const { tableId1, tableId2, rowId1, rowId2 } = oldLinkDocInfo() const linkDoc = await db.get(oldLinkDocID()) diff --git a/packages/server/src/sdk/app/rows/search/sqs.ts b/packages/server/src/sdk/app/rows/search/sqs.ts index 174ecc0e38..0720600a15 100644 --- a/packages/server/src/sdk/app/rows/search/sqs.ts +++ b/packages/server/src/sdk/app/rows/search/sqs.ts @@ -18,6 +18,7 @@ import { buildInternalRelationships, sqlOutputProcessing, } from "../../../../api/controllers/row/utils" +import { mapToUserColumn, USER_COLUMN_PREFIX } from "../../tables/internal/sqs" import sdk from "../../../index" import { context, @@ -35,8 +36,10 @@ import { getRelationshipColumns, getTableIDList, } from "./filters" +import { dataFilters } from "@budibase/shared-core" const builder = new sql.Sql(SqlClient.SQL_LITE) +const NO_SUCH_COLUMN_REGEX = new RegExp(`no such colum.+${USER_COLUMN_PREFIX}`) function buildInternalFieldList( table: Table, @@ -59,7 +62,7 @@ function buildInternalFieldList( buildInternalFieldList(relatedTable, tables, { relationships: false }) ) } else { - fieldList.push(`${table._id}.${col.name}`) + fieldList.push(`${table._id}.${mapToUserColumn(col.name)}`) } } return fieldList @@ -90,6 +93,34 @@ function cleanupFilters( ) ) + // generate a map of all possible column names (these can be duplicated across tables + // the map of them will always be the same + const userColumnMap: Record = {} + allTables.forEach(table => + Object.keys(table.schema).forEach( + key => (userColumnMap[key] = mapToUserColumn(key)) + ) + ) + + // update the keys of filters to manage user columns + const keyInAnyTable = (key: string): boolean => + allTables.some(table => table.schema[key]) + + const splitter = new dataFilters.ColumnSplitter(allTables) + for (const filter of Object.values(filters)) { + for (const key of Object.keys(filter)) { + const { numberPrefix, relationshipPrefix, column } = splitter.run(key) + if (keyInAnyTable(column)) { + filter[ + `${numberPrefix || ""}${relationshipPrefix || ""}${mapToUserColumn( + column + )}` + ] = filter[key] + delete filter[key] + } + } + } + return filters } @@ -106,6 +137,25 @@ function buildTableMap(tables: Table[]) { return tableMap } +function reverseUserColumnMapping(rows: Row[]) { + const prefixLength = USER_COLUMN_PREFIX.length + return rows.map(row => { + const finalRow: Row = {} + for (let key of Object.keys(row)) { + // it should be the first prefix + const index = key.indexOf(USER_COLUMN_PREFIX) + if (index !== -1) { + // cut out the prefix + const newKey = key.slice(0, index) + key.slice(index + prefixLength) + finalRow[newKey] = row[key] + } else { + finalRow[key] = row[key] + } + } + return finalRow + }) +} + function runSqlQuery(json: QueryJson, tables: Table[]): Promise function runSqlQuery( json: QueryJson, @@ -147,9 +197,10 @@ async function runSqlQuery( const response = await alias.queryWithAliasing(json, processSQLQuery) if (opts?.countTotalRows) { return processRowCountResponse(response) - } else { - return response + } else if (Array.isArray(response)) { + return reverseUserColumnMapping(response) } + return response } export async function search( @@ -185,6 +236,7 @@ export async function search( meta: { table, tables: allTablesMap, + columnPrefix: USER_COLUMN_PREFIX, }, resource: { fields: buildInternalFieldList(table, allTables), @@ -197,7 +249,7 @@ export async function search( const sortType = sortField.type === FieldType.NUMBER ? SortType.NUMBER : SortType.STRING request.sort = { - [sortField.name]: { + [mapToUserColumn(sortField.name)]: { direction: params.sortOrder || SortOrder.ASCENDING, type: sortType as SortType, }, @@ -278,7 +330,10 @@ export async function search( return response } catch (err: any) { const msg = typeof err === "string" ? err : err.message - if (err.status === 404 && msg?.includes(SQLITE_DESIGN_DOC_ID)) { + const syncAndRepeat = + (err.status === 400 && msg?.match(NO_SUCH_COLUMN_REGEX)) || + (err.status === 404 && msg?.includes(SQLITE_DESIGN_DOC_ID)) + if (syncAndRepeat) { await sdk.tables.sqs.syncDefinition() return search(options, table) } diff --git a/packages/server/src/sdk/app/tables/internal/sqs.ts b/packages/server/src/sdk/app/tables/internal/sqs.ts index 4819b9d8d5..f892a9c6c8 100644 --- a/packages/server/src/sdk/app/tables/internal/sqs.ts +++ b/packages/server/src/sdk/app/tables/internal/sqs.ts @@ -62,10 +62,18 @@ function buildRelationshipDefinitions( } } +export const USER_COLUMN_PREFIX = "data_" + +// utility function to denote that columns in SQLite are mapped to avoid overlap issues +// the overlaps can occur due to case insensitivity and some of the columns which Budibase requires +export function mapToUserColumn(key: string) { + return `${USER_COLUMN_PREFIX}${key}` +} + // this can generate relationship tables as part of the mapping function mapTable(table: Table): SQLiteTables { const tables: SQLiteTables = {} - const fields: Record = {} + const fields: Record = {} for (let [key, column] of Object.entries(table.schema)) { // relationships should be handled differently if (column.type === FieldType.LINK) { @@ -78,7 +86,10 @@ function mapTable(table: Table): SQLiteTables { if (!FieldTypeMap[column.type]) { throw new Error(`Unable to map type "${column.type}" to SQLite type`) } - fields[key] = FieldTypeMap[column.type] + fields[mapToUserColumn(key)] = { + field: key, + type: FieldTypeMap[column.type], + } } // there are some extra columns to map - add these in const constantMap: Record = {} diff --git a/packages/shared-core/src/constants/index.ts b/packages/shared-core/src/constants/index.ts index c9d1a8fc8f..0713b5d2f8 100644 --- a/packages/shared-core/src/constants/index.ts +++ b/packages/shared-core/src/constants/index.ts @@ -164,14 +164,17 @@ export const InvalidFileExtensions = [ export enum BpmCorrelationKey { ONBOARDING = "budibase:onboarding:correlationkey", + VERIFY_SSO_LOGIN = "budibase:verify_sso_login:correlationkey", } export enum BpmInstanceKey { ONBOARDING = "budibase:onboarding:instancekey", + VERIFY_SSO_LOGIN = "budibase:verify_sso_login:instancekey", } export enum BpmStatusKey { ONBOARDING = "budibase:onboarding:status", + VERIFY_SSO_LOGIN = "budibase:verify_sso_login:status", } export enum BpmStatusValue { diff --git a/packages/shared-core/src/filters.ts b/packages/shared-core/src/filters.ts index 443dbbce8d..3c6901e195 100644 --- a/packages/shared-core/src/filters.ts +++ b/packages/shared-core/src/filters.ts @@ -13,6 +13,7 @@ import { RowSearchParams, EmptyFilterOption, SearchResponse, + Table, } from "@budibase/types" import dayjs from "dayjs" import { OperatorOptions, SqlNumberTypeRangeMap } from "./constants" @@ -131,13 +132,72 @@ const cleanupQuery = (query: SearchFilters) => { * Removes a numeric prefix on field names designed to give fields uniqueness */ export const removeKeyNumbering = (key: string): string => { + return getKeyNumbering(key).key +} + +/** + * Gets the part of the keys, returning the numeric prefix and the field name + */ +export const getKeyNumbering = ( + key: string +): { prefix?: string; key: string } => { if (typeof key === "string" && key.match(/\d[0-9]*:/g) != null) { const parts = key.split(":") // remove the number - parts.shift() - return parts.join(":") + const number = parts.shift() + return { prefix: `${number}:`, key: parts.join(":") } } else { - return key + return { key } + } +} + +/** + * Generates a splitter which can be used to split columns from a context into + * their components (number prefix, relationship column/table, column name) + */ +export class ColumnSplitter { + tableNames: string[] + tableIds: string[] + relationshipColumnNames: string[] + relationships: string[] + + constructor(tables: Table[]) { + this.tableNames = tables.map(table => table.name) + this.tableIds = tables.map(table => table._id!) + this.relationshipColumnNames = tables.flatMap(table => + Object.keys(table.schema).filter( + columnName => table.schema[columnName].type === FieldType.LINK + ) + ) + this.relationships = this.tableNames + .concat(this.tableIds) + .concat(this.relationshipColumnNames) + // sort by length - makes sure there's no mis-matches due to similarities (sub column names) + .sort((a, b) => b.length - a.length) + } + + run(key: string): { + numberPrefix?: string + relationshipPrefix?: string + column: string + } { + let { prefix, key: splitKey } = getKeyNumbering(key) + let relationship: string | undefined + for (let possibleRelationship of this.relationships) { + const withDot = `${possibleRelationship}.` + if (splitKey.startsWith(withDot)) { + const finalKeyParts = splitKey.split(withDot) + finalKeyParts.shift() + relationship = withDot + splitKey = finalKeyParts.join(".") + break + } + } + return { + numberPrefix: prefix, + relationshipPrefix: relationship, + column: splitKey, + } } } diff --git a/packages/types/src/sdk/search.ts b/packages/types/src/sdk/search.ts index e5cbccf5c1..c40f1c3b84 100644 --- a/packages/types/src/sdk/search.ts +++ b/packages/types/src/sdk/search.ts @@ -122,6 +122,8 @@ export interface QueryJson { table: Table tables?: Record renamed?: RenameColumn + // can specify something that columns could be prefixed with + columnPrefix?: string } extra?: { idFilter?: SearchFilters diff --git a/scripts/deploy-camunda.sh b/scripts/deploy-camunda.sh index 7059b6f072..90400a0449 100755 --- a/scripts/deploy-camunda.sh +++ b/scripts/deploy-camunda.sh @@ -23,6 +23,7 @@ echo "deploy processes..." zbctl deploy resource offboarding.bpmn --insecure zbctl deploy resource onboarding.bpmn --insecure zbctl deploy resource free_trial.bpmn --insecure +zbctl deploy resource verify_sso_login.bpmn --insecure cd ../../../../../budibase/packages/account-portal/packages/server diff --git a/yarn.lock b/yarn.lock index 4cc63badcb..c060cc9da3 100644 --- a/yarn.lock +++ b/yarn.lock @@ -16917,7 +16917,7 @@ open@^8.0.0, open@^8.4.0, open@~8.4.0: is-docker "^2.1.1" is-wsl "^2.2.0" -openai@4.52.1: +openai@^4.52.1: version "4.52.1" resolved "https://registry.yarnpkg.com/openai/-/openai-4.52.1.tgz#44acc362a844fa2927b0cfa1fb70fb51e388af65" integrity sha512-kv2hevAWZZ3I/vd2t8znGO2rd8wkowncsfcYpo8i+wU9ML+JEcdqiViANXXjWWGjIhajFNixE6gOY1fEgqILAg== @@ -21786,16 +21786,16 @@ typescript-eslint@^7.3.1: "@typescript-eslint/eslint-plugin" "7.3.1" "@typescript-eslint/parser" "7.3.1" +typescript@5.2.2, "typescript@>=3 < 6": + version "5.2.2" + resolved "https://registry.yarnpkg.com/typescript/-/typescript-5.2.2.tgz#5ebb5e5a5b75f085f22bc3f8460fba308310fa78" + integrity sha512-mI4WrpHsbCIcwT9cF4FZvr80QUeKvsUsUvKDoR+X/7XHQH98xYD8YHZg7ANtz2GtZt/CBq2QJ0thkGJMHfqc1w== + typescript@5.5.2: version "5.5.2" resolved "https://registry.yarnpkg.com/typescript/-/typescript-5.5.2.tgz#c26f023cb0054e657ce04f72583ea2d85f8d0507" integrity sha512-NcRtPEOsPFFWjobJEtfihkLCZCXZt/os3zf8nTxjVH3RvTSxjrCamJpbExGvYOF+tFHc3pA65qpdwPbzjohhew== -"typescript@>=3 < 6": - version "5.2.2" - resolved "https://registry.yarnpkg.com/typescript/-/typescript-5.2.2.tgz#5ebb5e5a5b75f085f22bc3f8460fba308310fa78" - integrity sha512-mI4WrpHsbCIcwT9cF4FZvr80QUeKvsUsUvKDoR+X/7XHQH98xYD8YHZg7ANtz2GtZt/CBq2QJ0thkGJMHfqc1w== - typescript@^3.9.10, typescript@^3.9.5, typescript@^3.9.7: version "3.9.10" resolved "https://registry.yarnpkg.com/typescript/-/typescript-3.9.10.tgz#70f3910ac7a51ed6bef79da7800690b19bf778b8"