diff --git a/lerna.json b/lerna.json index 9d04750a0d..0efaf75283 100644 --- a/lerna.json +++ b/lerna.json @@ -1,5 +1,5 @@ { - "version": "2.29.0", + "version": "2.29.1", "npmClient": "yarn", "packages": [ "packages/*", diff --git a/packages/account-portal b/packages/account-portal index 247f56d455..b600cca314 160000 --- a/packages/account-portal +++ b/packages/account-portal @@ -1 +1 @@ -Subproject commit 247f56d455abbd64da17d865275ed978f577549f +Subproject commit b600cca314a5cc9971e44d46047d1a0019b46b08 diff --git a/packages/backend-core/src/constants/db.ts b/packages/backend-core/src/constants/db.ts index 2fd713119b..3085b91ef1 100644 --- a/packages/backend-core/src/constants/db.ts +++ b/packages/backend-core/src/constants/db.ts @@ -72,4 +72,4 @@ export const DEFAULT_JOBS_TABLE_ID = "ta_bb_jobs" export const DEFAULT_INVENTORY_TABLE_ID = "ta_bb_inventory" export const DEFAULT_EXPENSES_TABLE_ID = "ta_bb_expenses" export const DEFAULT_EMPLOYEE_TABLE_ID = "ta_bb_employee" -export const DEFAULT_BB_DATASOURCE_ID = "datasource_internal_bb_default" +export { DEFAULT_BB_DATASOURCE_ID } from "@budibase/shared-core" diff --git a/packages/backend-core/src/db/constants.ts b/packages/backend-core/src/db/constants.ts index bfa7595d62..69c98fe569 100644 --- a/packages/backend-core/src/db/constants.ts +++ b/packages/backend-core/src/db/constants.ts @@ -1,14 +1,5 @@ -export const CONSTANT_INTERNAL_ROW_COLS = [ - "_id", - "_rev", - "type", - "createdAt", - "updatedAt", - "tableId", -] as const - -export const CONSTANT_EXTERNAL_ROW_COLS = ["_id", "_rev", "tableId"] as const - -export function isInternalColumnName(name: string): boolean { - return (CONSTANT_INTERNAL_ROW_COLS as readonly string[]).includes(name) -} +export { + CONSTANT_INTERNAL_ROW_COLS, + CONSTANT_EXTERNAL_ROW_COLS, + isInternalColumnName, +} from "@budibase/shared-core" diff --git a/packages/backend-core/src/sql/sqlTable.ts b/packages/backend-core/src/sql/sqlTable.ts index 09f9908baa..bdc8a3dd69 100644 --- a/packages/backend-core/src/sql/sqlTable.ts +++ b/packages/backend-core/src/sql/sqlTable.ts @@ -109,8 +109,10 @@ function generateSchema( const { tableName } = breakExternalTableId(column.tableId) // @ts-ignore const relatedTable = tables[tableName] - if (!relatedTable) { - throw new Error("Referenced table doesn't exist") + if (!relatedTable || !relatedTable.primary) { + throw new Error( + "Referenced table doesn't exist or has no primary keys" + ) } const relatedPrimary = relatedTable.primary[0] const externalType = relatedTable.schema[relatedPrimary].externalType diff --git a/packages/backend-core/src/sql/utils.ts b/packages/backend-core/src/sql/utils.ts index 2d9b289417..45ab510948 100644 --- a/packages/backend-core/src/sql/utils.ts +++ b/packages/backend-core/src/sql/utils.ts @@ -55,10 +55,7 @@ export function buildExternalTableId(datasourceId: string, tableName: string) { return `${datasourceId}${DOUBLE_SEPARATOR}${tableName}` } -export function breakExternalTableId(tableId: string | undefined) { - if (!tableId) { - return {} - } +export function breakExternalTableId(tableId: string) { const parts = tableId.split(DOUBLE_SEPARATOR) let datasourceId = parts.shift() // if they need joined @@ -67,6 +64,9 @@ export function breakExternalTableId(tableId: string | undefined) { if (tableName.includes(ENCODED_SPACE)) { tableName = decodeURIComponent(tableName) } + if (!datasourceId || !tableName) { + throw new Error("Unable to get datasource/table name from table ID") + } return { datasourceId, tableName } } diff --git a/packages/builder/src/components/backend/DataTable/modals/CreateEditColumn.svelte b/packages/builder/src/components/backend/DataTable/modals/CreateEditColumn.svelte index 17ecd8f844..d79eedd194 100644 --- a/packages/builder/src/components/backend/DataTable/modals/CreateEditColumn.svelte +++ b/packages/builder/src/components/backend/DataTable/modals/CreateEditColumn.svelte @@ -17,6 +17,8 @@ SWITCHABLE_TYPES, ValidColumnNameRegex, helpers, + CONSTANT_INTERNAL_ROW_COLS, + CONSTANT_EXTERNAL_ROW_COLS, } from "@budibase/shared-core" import { createEventDispatcher, getContext, onMount } from "svelte" import { cloneDeep } from "lodash/fp" @@ -52,7 +54,6 @@ const DATE_TYPE = FieldType.DATETIME const dispatch = createEventDispatcher() - const PROHIBITED_COLUMN_NAMES = ["type", "_id", "_rev", "tableId"] const { dispatch: gridDispatch, rows } = getContext("grid") export let field @@ -487,20 +488,27 @@ }) } const newError = {} + const prohibited = externalTable + ? CONSTANT_EXTERNAL_ROW_COLS + : CONSTANT_INTERNAL_ROW_COLS if (!externalTable && fieldInfo.name?.startsWith("_")) { newError.name = `Column name cannot start with an underscore.` } else if (fieldInfo.name && !fieldInfo.name.match(ValidColumnNameRegex)) { newError.name = `Illegal character; must be alpha-numeric.` - } else if (PROHIBITED_COLUMN_NAMES.some(name => fieldInfo.name === name)) { - newError.name = `${PROHIBITED_COLUMN_NAMES.join( + } else if ( + prohibited.some( + name => fieldInfo?.name?.toLowerCase() === name.toLowerCase() + ) + ) { + newError.name = `${prohibited.join( ", " - )} are not allowed as column names` + )} are not allowed as column names - case insensitive.` } else if (inUse($tables.selected, fieldInfo.name, originalName)) { newError.name = `Column name already in use.` } if (fieldInfo.type === FieldType.AUTO && !fieldInfo.subtype) { - newError.subtype = `Auto Column requires a type` + newError.subtype = `Auto Column requires a type.` } if (fieldInfo.fieldName && fieldInfo.tableId) { diff --git a/packages/builder/src/components/design/settings/controls/ButtonActionEditor/actions/DeleteRow.svelte b/packages/builder/src/components/design/settings/controls/ButtonActionEditor/actions/DeleteRow.svelte index 5b7844ce53..fd3521d597 100644 --- a/packages/builder/src/components/design/settings/controls/ButtonActionEditor/actions/DeleteRow.svelte +++ b/packages/builder/src/components/design/settings/controls/ButtonActionEditor/actions/DeleteRow.svelte @@ -53,6 +53,12 @@ placeholder="Are you sure you want to delete?" bind:value={parameters.confirmText} /> + + + + + + {/if} diff --git a/packages/builder/src/components/design/settings/controls/ButtonActionEditor/actions/DuplicateRow.svelte b/packages/builder/src/components/design/settings/controls/ButtonActionEditor/actions/DuplicateRow.svelte index 3b4a7c2d38..b6cdd663fd 100644 --- a/packages/builder/src/components/design/settings/controls/ButtonActionEditor/actions/DuplicateRow.svelte +++ b/packages/builder/src/components/design/settings/controls/ButtonActionEditor/actions/DuplicateRow.svelte @@ -83,6 +83,12 @@ placeholder="Are you sure you want to duplicate this row?" bind:value={parameters.confirmText} /> + + + + + + {/if} diff --git a/packages/builder/src/components/design/settings/controls/ButtonActionEditor/actions/ExecuteQuery.svelte b/packages/builder/src/components/design/settings/controls/ButtonActionEditor/actions/ExecuteQuery.svelte index 54295d8b0f..43797f6369 100644 --- a/packages/builder/src/components/design/settings/controls/ButtonActionEditor/actions/ExecuteQuery.svelte +++ b/packages/builder/src/components/design/settings/controls/ButtonActionEditor/actions/ExecuteQuery.svelte @@ -74,6 +74,18 @@ placeholder="Are you sure you want to execute this query?" bind:value={parameters.confirmText} /> + + + + {/if} {#if query?.parameters?.length > 0} diff --git a/packages/builder/src/components/design/settings/controls/ButtonActionEditor/actions/SaveRow.svelte b/packages/builder/src/components/design/settings/controls/ButtonActionEditor/actions/SaveRow.svelte index d834e9aac9..aed2618778 100644 --- a/packages/builder/src/components/design/settings/controls/ButtonActionEditor/actions/SaveRow.svelte +++ b/packages/builder/src/components/design/settings/controls/ButtonActionEditor/actions/SaveRow.svelte @@ -80,6 +80,12 @@ placeholder="Are you sure you want to save this row?" bind:value={parameters.confirmText} /> + + + + + + {/if} diff --git a/packages/client/src/components/overlay/ConfirmationDisplay.svelte b/packages/client/src/components/overlay/ConfirmationDisplay.svelte index e7a1046191..b96af502df 100644 --- a/packages/client/src/components/overlay/ConfirmationDisplay.svelte +++ b/packages/client/src/components/overlay/ConfirmationDisplay.svelte @@ -8,6 +8,8 @@ {$confirmationStore.text} diff --git a/packages/client/src/stores/confirmation.js b/packages/client/src/stores/confirmation.js index bb9a54386f..3fbf3d5deb 100644 --- a/packages/client/src/stores/confirmation.js +++ b/packages/client/src/stores/confirmation.js @@ -4,6 +4,8 @@ const initialState = { showConfirmation: false, title: null, text: null, + confirmButtonText: null, + cancelButtonText: null, onConfirm: null, onCancel: null, } @@ -11,11 +13,20 @@ const initialState = { const createConfirmationStore = () => { const store = writable(initialState) - const showConfirmation = (title, text, onConfirm, onCancel) => { + const showConfirmation = ( + title, + text, + onConfirm, + onCancel, + confirmButtonText, + cancelButtonText + ) => { store.set({ showConfirmation: true, title, text, + confirmButtonText, + cancelButtonText, onConfirm, onCancel, }) diff --git a/packages/client/src/utils/buttonActions.js b/packages/client/src/utils/buttonActions.js index bd220b8e85..8f0cb575a7 100644 --- a/packages/client/src/utils/buttonActions.js +++ b/packages/client/src/utils/buttonActions.js @@ -522,6 +522,7 @@ const confirmTextMap = { ["Execute Query"]: "Are you sure you want to execute this query?", ["Trigger Automation"]: "Are you sure you want to trigger this automation?", ["Prompt User"]: "Are you sure you want to continue?", + ["Duplicate Row"]: "Are you sure you want to duplicate this row?", } /** @@ -582,6 +583,11 @@ export const enrichButtonActions = (actions, context) => { const defaultTitleText = action["##eventHandlerType"] const customTitleText = action.parameters?.customTitleText || defaultTitleText + const cancelButtonText = + action.parameters?.cancelButtonText || "Cancel" + const confirmButtonText = + action.parameters?.confirmButtonText || "Confirm" + confirmationStore.actions.showConfirmation( customTitleText, confirmText, @@ -612,7 +618,9 @@ export const enrichButtonActions = (actions, context) => { }, () => { resolve(false) - } + }, + confirmButtonText, + cancelButtonText ) }) } diff --git a/packages/frontend-core/src/components/FilterBuilder.svelte b/packages/frontend-core/src/components/FilterBuilder.svelte index 0d254186f2..6d1e1fa502 100644 --- a/packages/frontend-core/src/components/FilterBuilder.svelte +++ b/packages/frontend-core/src/components/FilterBuilder.svelte @@ -18,7 +18,7 @@ import FilterUsers from "./FilterUsers.svelte" import { getFields } from "../utils/searchFields" - const { OperatorOptions } = Constants + const { OperatorOptions, DEFAULT_BB_DATASOURCE_ID } = Constants export let schemaFields export let filters = [] @@ -28,6 +28,23 @@ export let allowBindings = false export let filtersLabel = "Filters" + $: { + if ( + tables.find( + table => + table._id === datasource.tableId && + table.sourceId === DEFAULT_BB_DATASOURCE_ID + ) && + !schemaFields.some(field => field.name === "_id") + ) { + schemaFields = [ + ...schemaFields, + { name: "_id", type: "string" }, + { name: "_rev", type: "string" }, + ] + } + } + $: matchAny = filters?.find(filter => filter.operator === "allOr") != null $: onEmptyFilter = filters?.find(filter => filter.onEmptyFilter)?.onEmptyFilter ?? "all" @@ -35,7 +52,6 @@ $: fieldFilters = filters.filter( filter => filter.operator !== "allOr" && !filter.onEmptyFilter ) - const behaviourOptions = [ { value: "and", label: "Match all filters" }, { value: "or", label: "Match any filter" }, @@ -44,7 +60,6 @@ { value: "all", label: "Return all table rows" }, { value: "none", label: "Return no rows" }, ] - const context = getContext("context") $: fieldOptions = getFields(tables, schemaFields || [], { diff --git a/packages/frontend-core/src/constants.js b/packages/frontend-core/src/constants.js index 0d6261f5f8..e5869a3b98 100644 --- a/packages/frontend-core/src/constants.js +++ b/packages/frontend-core/src/constants.js @@ -1,7 +1,11 @@ /** * Operator options for lucene queries */ -export { OperatorOptions, SqlNumberTypeRangeMap } from "@budibase/shared-core" +export { + OperatorOptions, + SqlNumberTypeRangeMap, + DEFAULT_BB_DATASOURCE_ID, +} from "@budibase/shared-core" export { Feature as Features } from "@budibase/types" import { BpmCorrelationKey } from "@budibase/shared-core" import { FieldType, BBReferenceFieldSubType } from "@budibase/types" diff --git a/packages/server/src/api/controllers/row/ExternalRequest.ts b/packages/server/src/api/controllers/row/ExternalRequest.ts index 619a1e9548..1ce8d29e5f 100644 --- a/packages/server/src/api/controllers/row/ExternalRequest.ts +++ b/packages/server/src/api/controllers/row/ExternalRequest.ts @@ -72,92 +72,6 @@ export type ExternalRequestReturnType = ? number : { row: Row; table: Table } -function buildFilters( - id: string | undefined | string[], - filters: SearchFilters, - table: Table -) { - const primary = table.primary - // if passed in array need to copy for shifting etc - let idCopy: undefined | string | any[] = cloneDeep(id) - if (filters) { - // need to map over the filters and make sure the _id field isn't present - let prefix = 1 - for (let operator of Object.values(filters)) { - for (let field of Object.keys(operator || {})) { - if (dbCore.removeKeyNumbering(field) === "_id") { - if (primary) { - const parts = breakRowIdField(operator[field]) - for (let field of primary) { - operator[`${prefix}:${field}`] = parts.shift() - } - prefix++ - } - // make sure this field doesn't exist on any filter - delete operator[field] - } - } - } - } - // there is no id, just use the user provided filters - if (!idCopy || !table) { - return filters - } - // if used as URL parameter it will have been joined - if (!Array.isArray(idCopy)) { - idCopy = breakRowIdField(idCopy) - } - const equal: any = {} - if (primary && idCopy) { - for (let field of primary) { - // work through the ID and get the parts - equal[field] = idCopy.shift() - } - } - return { - equal, - } -} - -async function removeManyToManyRelationships( - rowId: string, - table: Table, - colName: string -) { - const tableId = table._id! - const filters = buildFilters(rowId, {}, table) - // safety check, if there are no filters on deletion bad things happen - if (Object.keys(filters).length !== 0) { - return getDatasourceAndQuery({ - endpoint: getEndpoint(tableId, Operation.DELETE), - body: { [colName]: null }, - filters, - meta: { - table, - }, - }) - } else { - return [] - } -} - -async function removeOneToManyRelationships(rowId: string, table: Table) { - const tableId = table._id! - const filters = buildFilters(rowId, {}, table) - // safety check, if there are no filters on deletion bad things happen - if (Object.keys(filters).length !== 0) { - return getDatasourceAndQuery({ - endpoint: getEndpoint(tableId, Operation.UPDATE), - filters, - meta: { - table, - }, - }) - } else { - return [] - } -} - /** * This function checks the incoming parameters to make sure all the inputs are * valid based on on the table schema. The main thing this is looking for is when a @@ -212,8 +126,8 @@ function getEndpoint(tableId: string | undefined, operation: string) { } const { datasourceId, tableName } = breakExternalTableId(tableId) return { - datasourceId: datasourceId!, - entityId: tableName!, + datasourceId: datasourceId, + entityId: tableName, operation: operation as Operation, } } @@ -240,6 +154,7 @@ export class ExternalRequest { private readonly tableId: string private datasource?: Datasource private tables: { [key: string]: Table } = {} + private tableList: Table[] constructor(operation: T, tableId: string, datasource?: Datasource) { this.operation = operation @@ -248,22 +163,134 @@ export class ExternalRequest { if (datasource && datasource.entities) { this.tables = datasource.entities } + this.tableList = Object.values(this.tables) + } + + private prepareFilters( + id: string | undefined | string[], + filters: SearchFilters, + table: Table + ): SearchFilters { + // replace any relationship columns initially, table names and relationship column names are acceptable + const relationshipColumns = sdk.rows.filters.getRelationshipColumns(table) + filters = sdk.rows.filters.updateFilterKeys( + filters, + relationshipColumns.map(({ name, definition }) => { + const { tableName } = breakExternalTableId(definition.tableId) + return { + original: name, + updated: tableName, + } + }) + ) + const primary = table.primary + // if passed in array need to copy for shifting etc + let idCopy: undefined | string | any[] = cloneDeep(id) + if (filters) { + // need to map over the filters and make sure the _id field isn't present + let prefix = 1 + for (let operator of Object.values(filters)) { + for (let field of Object.keys(operator || {})) { + if (dbCore.removeKeyNumbering(field) === "_id") { + if (primary) { + const parts = breakRowIdField(operator[field]) + for (let field of primary) { + operator[`${prefix}:${field}`] = parts.shift() + } + prefix++ + } + // make sure this field doesn't exist on any filter + delete operator[field] + } + } + } + } + // there is no id, just use the user provided filters + if (!idCopy || !table) { + return filters + } + // if used as URL parameter it will have been joined + if (!Array.isArray(idCopy)) { + idCopy = breakRowIdField(idCopy) + } + const equal: SearchFilters["equal"] = {} + if (primary && idCopy) { + for (let field of primary) { + // work through the ID and get the parts + equal[field] = idCopy.shift() + } + } + return { + equal, + } + } + + private async removeManyToManyRelationships( + rowId: string, + table: Table, + colName: string + ) { + const tableId = table._id! + const filters = this.prepareFilters(rowId, {}, table) + // safety check, if there are no filters on deletion bad things happen + if (Object.keys(filters).length !== 0) { + return getDatasourceAndQuery({ + endpoint: getEndpoint(tableId, Operation.DELETE), + body: { [colName]: null }, + filters, + meta: { + table, + }, + }) + } else { + return [] + } + } + + private async removeOneToManyRelationships(rowId: string, table: Table) { + const tableId = table._id! + const filters = this.prepareFilters(rowId, {}, table) + // safety check, if there are no filters on deletion bad things happen + if (Object.keys(filters).length !== 0) { + return getDatasourceAndQuery({ + endpoint: getEndpoint(tableId, Operation.UPDATE), + filters, + meta: { + table, + }, + }) + } else { + return [] + } } getTable(tableId: string | undefined): Table | undefined { if (!tableId) { - throw "Table ID is unknown, cannot find table" + throw new Error("Table ID is unknown, cannot find table") } const { tableName } = breakExternalTableId(tableId) - if (tableName) { - return this.tables[tableName] + return this.tables[tableName] + } + + // seeds the object with table and datasource information + async retrieveMetadata( + datasourceId: string + ): Promise<{ tables: Record; datasource: Datasource }> { + if (!this.datasource) { + this.datasource = await sdk.datasources.get(datasourceId) + if (!this.datasource || !this.datasource.entities) { + throw "No tables found, fetch tables before query." + } + this.tables = this.datasource.entities + this.tableList = Object.values(this.tables) } + return { tables: this.tables, datasource: this.datasource } } async getRow(table: Table, rowId: string): Promise { const response = await getDatasourceAndQuery({ endpoint: getEndpoint(table._id!, Operation.READ), - filters: buildFilters(rowId, {}, table), + filters: this.prepareFilters(rowId, {}, table), meta: { table, }, @@ -296,9 +323,7 @@ export class ExternalRequest { if (field.type === FieldType.NUMBER && !isNaN(parseFloat(row[key]))) { newRow[key] = parseFloat(row[key]) } else if (field.type === FieldType.LINK) { - const { tableName: linkTableName } = breakExternalTableId( - field?.tableId - ) + const { tableName: linkTableName } = breakExternalTableId(field.tableId) // table has to exist for many to many if (!linkTableName || !this.tables[linkTableName]) { continue @@ -379,9 +404,6 @@ export class ExternalRequest { [key: string]: { rows: Row[]; isMany: boolean; tableId: string } } = {} const { tableName } = breakExternalTableId(tableId) - if (!tableName) { - return related - } const table = this.tables[tableName] // @ts-ignore const primaryKey = table.primary[0] @@ -514,7 +536,7 @@ export class ExternalRequest { endpoint: getEndpoint(tableId, operation), // if we're doing many relationships then we're writing, only one response body, - filters: buildFilters(id, {}, linkTable), + filters: this.prepareFilters(id, {}, linkTable), meta: { table: linkTable, }, @@ -538,8 +560,8 @@ export class ExternalRequest { for (let row of rows) { const rowId = generateIdForRow(row, table) const promise: Promise = isMany - ? removeManyToManyRelationships(rowId, table, colName) - : removeOneToManyRelationships(rowId, table) + ? this.removeManyToManyRelationships(rowId, table, colName) + : this.removeOneToManyRelationships(rowId, table) if (promise) { promises.push(promise) } @@ -562,12 +584,12 @@ export class ExternalRequest { rows.map(row => { const rowId = generateIdForRow(row, table) return isMany - ? removeManyToManyRelationships( + ? this.removeManyToManyRelationships( rowId, table, relationshipColumn.fieldName ) - : removeOneToManyRelationships(rowId, table) + : this.removeOneToManyRelationships(rowId, table) }) ) } @@ -575,21 +597,21 @@ export class ExternalRequest { async run(config: RunConfig): Promise> { const { operation, tableId } = this - let { datasourceId, tableName } = breakExternalTableId(tableId) - if (!tableName) { - throw "Unable to run without a table name" + if (!tableId) { + throw new Error("Unable to run without a table ID") } - if (!this.datasource) { - this.datasource = await sdk.datasources.get(datasourceId!) - if (!this.datasource || !this.datasource.entities) { - throw "No tables found, fetch tables before query." - } - this.tables = this.datasource.entities + let { datasourceId, tableName } = breakExternalTableId(tableId) + let datasource = this.datasource + if (!datasource) { + const { datasource: ds } = await this.retrieveMetadata(datasourceId) + datasource = ds } const table = this.tables[tableName] - let isSql = isSQL(this.datasource) + let isSql = isSQL(datasource) if (!table) { - throw `Unable to process query, table "${tableName}" not defined.` + throw new Error( + `Unable to process query, table "${tableName}" not defined.` + ) } // look for specific components of config which may not be considered acceptable let { id, row, filters, sort, paginate, rows } = cleanupConfig( @@ -612,7 +634,7 @@ export class ExternalRequest { break } } - filters = buildFilters(id, filters || {}, table) + filters = this.prepareFilters(id, filters || {}, table) const relationships = buildExternalRelationships(table, this.tables) const incRelationships = @@ -660,7 +682,11 @@ export class ExternalRequest { body: row || rows, // pass an id filter into extra, purely for mysql/returning extra: { - idFilter: buildFilters(id || generateIdForRow(row, table), {}, table), + idFilter: this.prepareFilters( + id || generateIdForRow(row, table), + {}, + table + ), }, meta: { table, diff --git a/packages/server/src/api/controllers/row/external.ts b/packages/server/src/api/controllers/row/external.ts index 5b12b5c207..126b11d0c1 100644 --- a/packages/server/src/api/controllers/row/external.ts +++ b/packages/server/src/api/controllers/row/external.ts @@ -136,10 +136,7 @@ export async function fetchEnrichedRow(ctx: UserCtx) { const id = ctx.params.rowId const tableId = utils.getTableId(ctx) const { datasourceId, tableName } = breakExternalTableId(tableId) - const datasource: Datasource = await sdk.datasources.get(datasourceId!) - if (!tableName) { - ctx.throw(400, "Unable to find table.") - } + const datasource: Datasource = await sdk.datasources.get(datasourceId) if (!datasource || !datasource.entities) { ctx.throw(400, "Datasource has not been configured for plus API.") } @@ -163,7 +160,7 @@ export async function fetchEnrichedRow(ctx: UserCtx) { } const links = row[fieldName] const linkedTableId = field.tableId - const linkedTableName = breakExternalTableId(linkedTableId).tableName! + const linkedTableName = breakExternalTableId(linkedTableId).tableName const linkedTable = tables[linkedTableName] // don't support composite keys right now const linkedIds = links.map((link: Row) => breakRowIdField(link._id!)[0]) diff --git a/packages/server/src/api/controllers/row/utils/basic.ts b/packages/server/src/api/controllers/row/utils/basic.ts index afb98d0255..bca2494ac3 100644 --- a/packages/server/src/api/controllers/row/utils/basic.ts +++ b/packages/server/src/api/controllers/row/utils/basic.ts @@ -99,7 +99,7 @@ export function basicProcessing({ row, tableName: table._id!, fieldName: internalColumn, - isLinked: false, + isLinked, }) } } diff --git a/packages/server/src/api/controllers/row/utils/sqlUtils.ts b/packages/server/src/api/controllers/row/utils/sqlUtils.ts index 6f7bdc7335..767916616c 100644 --- a/packages/server/src/api/controllers/row/utils/sqlUtils.ts +++ b/packages/server/src/api/controllers/row/utils/sqlUtils.ts @@ -2,6 +2,8 @@ import { DatasourcePlusQueryResponse, DSPlusOperation, FieldType, + isManyToOne, + isOneToMany, ManyToManyRelationshipFieldMetadata, RelationshipFieldMetadata, RelationshipsJson, @@ -93,12 +95,12 @@ export function buildExternalRelationships( ): RelationshipsJson[] { const relationships = [] for (let [fieldName, field] of Object.entries(table.schema)) { - if (field.type !== FieldType.LINK) { + if (field.type !== FieldType.LINK || !field.tableId) { continue } const { tableName: linkTableName } = breakExternalTableId(field.tableId) // no table to link to, this is not a valid relationships - if (!linkTableName || !tables[linkTableName]) { + if (!tables[linkTableName]) { continue } const linkTable = tables[linkTableName] @@ -110,7 +112,7 @@ export function buildExternalRelationships( // need to specify where to put this back into column: fieldName, } - if (isManyToMany(field)) { + if (isManyToMany(field) && field.through) { const { tableName: throughTableName } = breakExternalTableId( field.through ) @@ -120,7 +122,7 @@ export function buildExternalRelationships( definition.to = field.throughFrom || linkTable.primary[0] definition.fromPrimary = table.primary[0] definition.toPrimary = linkTable.primary[0] - } else { + } else if (isManyToOne(field) || isOneToMany(field)) { // if no foreign key specified then use the name of the field in other table definition.from = field.foreignKey || table.primary[0] definition.to = field.fieldName @@ -180,16 +182,18 @@ export function buildSqlFieldList( } let fields = extractRealFields(table) for (let field of Object.values(table.schema)) { - if (field.type !== FieldType.LINK || !opts?.relationships) { + if ( + field.type !== FieldType.LINK || + !opts?.relationships || + !field.tableId + ) { continue } const { tableName: linkTableName } = breakExternalTableId(field.tableId) - if (linkTableName) { - const linkTable = tables[linkTableName] - if (linkTable) { - const linkedFields = extractRealFields(linkTable, fields) - fields = fields.concat(linkedFields) - } + const linkTable = tables[linkTableName] + if (linkTable) { + const linkedFields = extractRealFields(linkTable, fields) + fields = fields.concat(linkedFields) } } return fields diff --git a/packages/server/src/api/controllers/table/external.ts b/packages/server/src/api/controllers/table/external.ts index f1b186c233..6ca8cdd82c 100644 --- a/packages/server/src/api/controllers/table/external.ts +++ b/packages/server/src/api/controllers/table/external.ts @@ -19,11 +19,14 @@ import { inputProcessing } from "../../../utilities/rowProcessor" function getDatasourceId(table: Table) { if (!table) { - throw "No table supplied" + throw new Error("No table supplied") } if (table.sourceId) { return table.sourceId } + if (!table._id) { + throw new Error("No table ID supplied") + } return breakExternalTableId(table._id).datasourceId } diff --git a/packages/server/src/api/routes/tests/search.spec.ts b/packages/server/src/api/routes/tests/search.spec.ts index be66253090..23c2ca819c 100644 --- a/packages/server/src/api/routes/tests/search.spec.ts +++ b/packages/server/src/api/routes/tests/search.spec.ts @@ -76,9 +76,9 @@ describe.each([ } }) - async function createTable(schema: TableSchema) { + async function createTable(schema: TableSchema, name?: string) { return await config.api.table.save( - tableForDatasource(datasource, { schema }) + tableForDatasource(datasource, { schema, name }) ) } @@ -274,55 +274,63 @@ describe.each([ }) describe("equal", () => { - it("successfully finds true row", () => - expectQuery({ equal: { isTrue: true } }).toMatchExactly([ + it("successfully finds true row", async () => { + await expectQuery({ equal: { isTrue: true } }).toMatchExactly([ { isTrue: true }, - ])) + ]) + }) - it("successfully finds false row", () => - expectQuery({ equal: { isTrue: false } }).toMatchExactly([ + it("successfully finds false row", async () => { + await expectQuery({ equal: { isTrue: false } }).toMatchExactly([ { isTrue: false }, - ])) + ]) + }) }) describe("notEqual", () => { - it("successfully finds false row", () => - expectQuery({ notEqual: { isTrue: true } }).toContainExactly([ + it("successfully finds false row", async () => { + await expectQuery({ notEqual: { isTrue: true } }).toContainExactly([ { isTrue: false }, - ])) + ]) + }) - it("successfully finds true row", () => - expectQuery({ notEqual: { isTrue: false } }).toContainExactly([ + it("successfully finds true row", async () => { + await expectQuery({ notEqual: { isTrue: false } }).toContainExactly([ { isTrue: true }, - ])) + ]) + }) }) describe("oneOf", () => { - it("successfully finds true row", () => - expectQuery({ oneOf: { isTrue: [true] } }).toContainExactly([ + it("successfully finds true row", async () => { + await expectQuery({ oneOf: { isTrue: [true] } }).toContainExactly([ { isTrue: true }, - ])) + ]) + }) - it("successfully finds false row", () => - expectQuery({ oneOf: { isTrue: [false] } }).toContainExactly([ + it("successfully finds false row", async () => { + await expectQuery({ oneOf: { isTrue: [false] } }).toContainExactly([ { isTrue: false }, - ])) + ]) + }) }) describe("sort", () => { - it("sorts ascending", () => - expectSearch({ + it("sorts ascending", async () => { + await expectSearch({ query: {}, sort: "isTrue", sortOrder: SortOrder.ASCENDING, - }).toMatchExactly([{ isTrue: false }, { isTrue: true }])) + }).toMatchExactly([{ isTrue: false }, { isTrue: true }]) + }) - it("sorts descending", () => - expectSearch({ + it("sorts descending", async () => { + await expectSearch({ query: {}, sort: "isTrue", sortOrder: SortOrder.DESCENDING, - }).toMatchExactly([{ isTrue: true }, { isTrue: false }])) + }).toMatchExactly([{ isTrue: true }, { isTrue: false }]) + }) }) }) @@ -676,191 +684,230 @@ describe.each([ }) describe("misc", () => { - it("should return all if no query is passed", () => - expectSearch({} as RowSearchParams).toContainExactly([ + it("should return all if no query is passed", async () => { + await expectSearch({} as RowSearchParams).toContainExactly([ { name: "foo" }, { name: "bar" }, - ])) + ]) + }) - it("should return all if empty query is passed", () => - expectQuery({}).toContainExactly([{ name: "foo" }, { name: "bar" }])) + it("should return all if empty query is passed", async () => { + await expectQuery({}).toContainExactly([ + { name: "foo" }, + { name: "bar" }, + ]) + }) - it("should return all if onEmptyFilter is RETURN_ALL", () => - expectQuery({ + it("should return all if onEmptyFilter is RETURN_ALL", async () => { + await expectQuery({ onEmptyFilter: EmptyFilterOption.RETURN_ALL, - }).toContainExactly([{ name: "foo" }, { name: "bar" }])) + }).toContainExactly([{ name: "foo" }, { name: "bar" }]) + }) - it("should return nothing if onEmptyFilter is RETURN_NONE", () => - expectQuery({ + it("should return nothing if onEmptyFilter is RETURN_NONE", async () => { + await expectQuery({ onEmptyFilter: EmptyFilterOption.RETURN_NONE, - }).toFindNothing()) + }).toFindNothing() + }) - it("should respect limit", () => - expectSearch({ limit: 1, paginate: true, query: {} }).toHaveLength(1)) + it("should respect limit", async () => { + await expectSearch({ + limit: 1, + paginate: true, + query: {}, + }).toHaveLength(1) + }) }) describe("equal", () => { - it("successfully finds a row", () => - expectQuery({ equal: { name: "foo" } }).toContainExactly([ + it("successfully finds a row", async () => { + await expectQuery({ equal: { name: "foo" } }).toContainExactly([ { name: "foo" }, - ])) + ]) + }) - it("fails to find nonexistent row", () => - expectQuery({ equal: { name: "none" } }).toFindNothing()) + it("fails to find nonexistent row", async () => { + await expectQuery({ equal: { name: "none" } }).toFindNothing() + }) - it("works as an or condition", () => - expectQuery({ + it("works as an or condition", async () => { + await expectQuery({ allOr: true, equal: { name: "foo" }, oneOf: { name: ["bar"] }, - }).toContainExactly([{ name: "foo" }, { name: "bar" }])) + }).toContainExactly([{ name: "foo" }, { name: "bar" }]) + }) - it("can have multiple values for same column", () => - expectQuery({ + it("can have multiple values for same column", async () => { + await expectQuery({ allOr: true, equal: { "1:name": "foo", "2:name": "bar" }, - }).toContainExactly([{ name: "foo" }, { name: "bar" }])) + }).toContainExactly([{ name: "foo" }, { name: "bar" }]) + }) }) describe("notEqual", () => { - it("successfully finds a row", () => - expectQuery({ notEqual: { name: "foo" } }).toContainExactly([ + it("successfully finds a row", async () => { + await expectQuery({ notEqual: { name: "foo" } }).toContainExactly([ { name: "bar" }, - ])) + ]) + }) - it("fails to find nonexistent row", () => - expectQuery({ notEqual: { name: "bar" } }).toContainExactly([ + it("fails to find nonexistent row", async () => { + await expectQuery({ notEqual: { name: "bar" } }).toContainExactly([ { name: "foo" }, - ])) + ]) + }) }) describe("oneOf", () => { - it("successfully finds a row", () => - expectQuery({ oneOf: { name: ["foo"] } }).toContainExactly([ + it("successfully finds a row", async () => { + await expectQuery({ oneOf: { name: ["foo"] } }).toContainExactly([ { name: "foo" }, - ])) + ]) + }) - it("fails to find nonexistent row", () => - expectQuery({ oneOf: { name: ["none"] } }).toFindNothing()) + it("fails to find nonexistent row", async () => { + await expectQuery({ oneOf: { name: ["none"] } }).toFindNothing() + }) }) describe("fuzzy", () => { - it("successfully finds a row", () => - expectQuery({ fuzzy: { name: "oo" } }).toContainExactly([ + it("successfully finds a row", async () => { + await expectQuery({ fuzzy: { name: "oo" } }).toContainExactly([ { name: "foo" }, - ])) + ]) + }) - it("fails to find nonexistent row", () => - expectQuery({ fuzzy: { name: "none" } }).toFindNothing()) + it("fails to find nonexistent row", async () => { + await expectQuery({ fuzzy: { name: "none" } }).toFindNothing() + }) }) describe("string", () => { - it("successfully finds a row", () => - expectQuery({ string: { name: "fo" } }).toContainExactly([ + it("successfully finds a row", async () => { + await expectQuery({ string: { name: "fo" } }).toContainExactly([ { name: "foo" }, - ])) + ]) + }) - it("fails to find nonexistent row", () => - expectQuery({ string: { name: "none" } }).toFindNothing()) + it("fails to find nonexistent row", async () => { + await expectQuery({ string: { name: "none" } }).toFindNothing() + }) - it("is case-insensitive", () => - expectQuery({ string: { name: "FO" } }).toContainExactly([ + it("is case-insensitive", async () => { + await expectQuery({ string: { name: "FO" } }).toContainExactly([ { name: "foo" }, - ])) + ]) + }) }) describe("range", () => { - it("successfully finds multiple rows", () => - expectQuery({ + it("successfully finds multiple rows", async () => { + await expectQuery({ range: { name: { low: "a", high: "z" } }, - }).toContainExactly([{ name: "bar" }, { name: "foo" }])) + }).toContainExactly([{ name: "bar" }, { name: "foo" }]) + }) - it("successfully finds a row with a high bound", () => - expectQuery({ + it("successfully finds a row with a high bound", async () => { + await expectQuery({ range: { name: { low: "a", high: "c" } }, - }).toContainExactly([{ name: "bar" }])) + }).toContainExactly([{ name: "bar" }]) + }) - it("successfully finds a row with a low bound", () => - expectQuery({ + it("successfully finds a row with a low bound", async () => { + await expectQuery({ range: { name: { low: "f", high: "z" } }, - }).toContainExactly([{ name: "foo" }])) + }).toContainExactly([{ name: "foo" }]) + }) - it("successfully finds no rows", () => - expectQuery({ + it("successfully finds no rows", async () => { + await expectQuery({ range: { name: { low: "g", high: "h" } }, - }).toFindNothing()) + }).toFindNothing() + }) !isLucene && - it("ignores low if it's an empty object", () => - expectQuery({ + it("ignores low if it's an empty object", async () => { + await expectQuery({ // @ts-ignore range: { name: { low: {}, high: "z" } }, - }).toContainExactly([{ name: "foo" }, { name: "bar" }])) + }).toContainExactly([{ name: "foo" }, { name: "bar" }]) + }) !isLucene && - it("ignores high if it's an empty object", () => - expectQuery({ + it("ignores high if it's an empty object", async () => { + await expectQuery({ // @ts-ignore range: { name: { low: "a", high: {} } }, - }).toContainExactly([{ name: "foo" }, { name: "bar" }])) + }).toContainExactly([{ name: "foo" }, { name: "bar" }]) + }) }) describe("empty", () => { - it("finds no empty rows", () => - expectQuery({ empty: { name: null } }).toFindNothing()) + it("finds no empty rows", async () => { + await expectQuery({ empty: { name: null } }).toFindNothing() + }) - it("should not be affected by when filter empty behaviour", () => - expectQuery({ + it("should not be affected by when filter empty behaviour", async () => { + await expectQuery({ empty: { name: null }, onEmptyFilter: EmptyFilterOption.RETURN_ALL, - }).toFindNothing()) + }).toFindNothing() + }) }) describe("notEmpty", () => { - it("finds all non-empty rows", () => - expectQuery({ notEmpty: { name: null } }).toContainExactly([ + it("finds all non-empty rows", async () => { + await expectQuery({ notEmpty: { name: null } }).toContainExactly([ { name: "foo" }, { name: "bar" }, - ])) + ]) + }) - it("should not be affected by when filter empty behaviour", () => - expectQuery({ + it("should not be affected by when filter empty behaviour", async () => { + await expectQuery({ notEmpty: { name: null }, onEmptyFilter: EmptyFilterOption.RETURN_NONE, - }).toContainExactly([{ name: "foo" }, { name: "bar" }])) + }).toContainExactly([{ name: "foo" }, { name: "bar" }]) + }) }) describe("sort", () => { - it("sorts ascending", () => - expectSearch({ + it("sorts ascending", async () => { + await expectSearch({ query: {}, sort: "name", sortOrder: SortOrder.ASCENDING, - }).toMatchExactly([{ name: "bar" }, { name: "foo" }])) + }).toMatchExactly([{ name: "bar" }, { name: "foo" }]) + }) - it("sorts descending", () => - expectSearch({ + it("sorts descending", async () => { + await expectSearch({ query: {}, sort: "name", sortOrder: SortOrder.DESCENDING, - }).toMatchExactly([{ name: "foo" }, { name: "bar" }])) + }).toMatchExactly([{ name: "foo" }, { name: "bar" }]) + }) describe("sortType STRING", () => { - it("sorts ascending", () => - expectSearch({ + it("sorts ascending", async () => { + await expectSearch({ query: {}, sort: "name", sortType: SortType.STRING, sortOrder: SortOrder.ASCENDING, - }).toMatchExactly([{ name: "bar" }, { name: "foo" }])) + }).toMatchExactly([{ name: "bar" }, { name: "foo" }]) + }) - it("sorts descending", () => - expectSearch({ + it("sorts descending", async () => { + await expectSearch({ query: {}, sort: "name", sortType: SortType.STRING, sortOrder: SortOrder.DESCENDING, - }).toMatchExactly([{ name: "foo" }, { name: "bar" }])) + }).toMatchExactly([{ name: "foo" }, { name: "bar" }]) + }) }) }) }) @@ -874,97 +921,119 @@ describe.each([ }) describe("equal", () => { - it("successfully finds a row", () => - expectQuery({ equal: { age: 1 } }).toContainExactly([{ age: 1 }])) + it("successfully finds a row", async () => { + await expectQuery({ equal: { age: 1 } }).toContainExactly([{ age: 1 }]) + }) - it("fails to find nonexistent row", () => - expectQuery({ equal: { age: 2 } }).toFindNothing()) + it("fails to find nonexistent row", async () => { + await expectQuery({ equal: { age: 2 } }).toFindNothing() + }) }) describe("notEqual", () => { - it("successfully finds a row", () => - expectQuery({ notEqual: { age: 1 } }).toContainExactly([{ age: 10 }])) + it("successfully finds a row", async () => { + await expectQuery({ notEqual: { age: 1 } }).toContainExactly([ + { age: 10 }, + ]) + }) - it("fails to find nonexistent row", () => - expectQuery({ notEqual: { age: 10 } }).toContainExactly([{ age: 1 }])) + it("fails to find nonexistent row", async () => { + await expectQuery({ notEqual: { age: 10 } }).toContainExactly([ + { age: 1 }, + ]) + }) }) describe("oneOf", () => { - it("successfully finds a row", () => - expectQuery({ oneOf: { age: [1] } }).toContainExactly([{ age: 1 }])) + it("successfully finds a row", async () => { + await expectQuery({ oneOf: { age: [1] } }).toContainExactly([ + { age: 1 }, + ]) + }) - it("fails to find nonexistent row", () => - expectQuery({ oneOf: { age: [2] } }).toFindNothing()) + it("fails to find nonexistent row", async () => { + await expectQuery({ oneOf: { age: [2] } }).toFindNothing() + }) }) describe("range", () => { - it("successfully finds a row", () => - expectQuery({ + it("successfully finds a row", async () => { + await expectQuery({ range: { age: { low: 1, high: 5 } }, - }).toContainExactly([{ age: 1 }])) + }).toContainExactly([{ age: 1 }]) + }) - it("successfully finds multiple rows", () => - expectQuery({ + it("successfully finds multiple rows", async () => { + await expectQuery({ range: { age: { low: 1, high: 10 } }, - }).toContainExactly([{ age: 1 }, { age: 10 }])) + }).toContainExactly([{ age: 1 }, { age: 10 }]) + }) - it("successfully finds a row with a high bound", () => - expectQuery({ + it("successfully finds a row with a high bound", async () => { + await expectQuery({ range: { age: { low: 5, high: 10 } }, - }).toContainExactly([{ age: 10 }])) + }).toContainExactly([{ age: 10 }]) + }) - it("successfully finds no rows", () => - expectQuery({ + it("successfully finds no rows", async () => { + await expectQuery({ range: { age: { low: 5, high: 9 } }, - }).toFindNothing()) + }).toFindNothing() + }) // We never implemented half-open ranges in Lucene. !isLucene && - it("can search using just a low value", () => - expectQuery({ + it("can search using just a low value", async () => { + await expectQuery({ range: { age: { low: 5 } }, - }).toContainExactly([{ age: 10 }])) + }).toContainExactly([{ age: 10 }]) + }) // We never implemented half-open ranges in Lucene. !isLucene && - it("can search using just a high value", () => - expectQuery({ + it("can search using just a high value", async () => { + await expectQuery({ range: { age: { high: 5 } }, - }).toContainExactly([{ age: 1 }])) + }).toContainExactly([{ age: 1 }]) + }) }) describe("sort", () => { - it("sorts ascending", () => - expectSearch({ + it("sorts ascending", async () => { + await expectSearch({ query: {}, sort: "age", sortOrder: SortOrder.ASCENDING, - }).toMatchExactly([{ age: 1 }, { age: 10 }])) + }).toMatchExactly([{ age: 1 }, { age: 10 }]) + }) - it("sorts descending", () => - expectSearch({ + it("sorts descending", async () => { + await expectSearch({ query: {}, sort: "age", sortOrder: SortOrder.DESCENDING, - }).toMatchExactly([{ age: 10 }, { age: 1 }])) + }).toMatchExactly([{ age: 10 }, { age: 1 }]) + }) }) describe("sortType NUMBER", () => { - it("sorts ascending", () => - expectSearch({ + it("sorts ascending", async () => { + await expectSearch({ query: {}, sort: "age", sortType: SortType.NUMBER, sortOrder: SortOrder.ASCENDING, - }).toMatchExactly([{ age: 1 }, { age: 10 }])) + }).toMatchExactly([{ age: 1 }, { age: 10 }]) + }) - it("sorts descending", () => - expectSearch({ + it("sorts descending", async () => { + await expectSearch({ query: {}, sort: "age", sortType: SortType.NUMBER, sortOrder: SortOrder.DESCENDING, - }).toMatchExactly([{ age: 10 }, { age: 1 }])) + }).toMatchExactly([{ age: 10 }, { age: 1 }]) + }) }) }) @@ -984,104 +1053,120 @@ describe.each([ }) describe("equal", () => { - it("successfully finds a row", () => - expectQuery({ equal: { dob: JAN_1ST } }).toContainExactly([ + it("successfully finds a row", async () => { + await expectQuery({ equal: { dob: JAN_1ST } }).toContainExactly([ { dob: JAN_1ST }, - ])) + ]) + }) - it("fails to find nonexistent row", () => - expectQuery({ equal: { dob: JAN_2ND } }).toFindNothing()) + it("fails to find nonexistent row", async () => { + await expectQuery({ equal: { dob: JAN_2ND } }).toFindNothing() + }) }) describe("notEqual", () => { - it("successfully finds a row", () => - expectQuery({ notEqual: { dob: JAN_1ST } }).toContainExactly([ + it("successfully finds a row", async () => { + await expectQuery({ notEqual: { dob: JAN_1ST } }).toContainExactly([ { dob: JAN_10TH }, - ])) + ]) + }) - it("fails to find nonexistent row", () => - expectQuery({ notEqual: { dob: JAN_10TH } }).toContainExactly([ + it("fails to find nonexistent row", async () => { + await expectQuery({ notEqual: { dob: JAN_10TH } }).toContainExactly([ { dob: JAN_1ST }, - ])) + ]) + }) }) describe("oneOf", () => { - it("successfully finds a row", () => - expectQuery({ oneOf: { dob: [JAN_1ST] } }).toContainExactly([ + it("successfully finds a row", async () => { + await expectQuery({ oneOf: { dob: [JAN_1ST] } }).toContainExactly([ { dob: JAN_1ST }, - ])) + ]) + }) - it("fails to find nonexistent row", () => - expectQuery({ oneOf: { dob: [JAN_2ND] } }).toFindNothing()) + it("fails to find nonexistent row", async () => { + await expectQuery({ oneOf: { dob: [JAN_2ND] } }).toFindNothing() + }) }) describe("range", () => { - it("successfully finds a row", () => - expectQuery({ + it("successfully finds a row", async () => { + await expectQuery({ range: { dob: { low: JAN_1ST, high: JAN_5TH } }, - }).toContainExactly([{ dob: JAN_1ST }])) + }).toContainExactly([{ dob: JAN_1ST }]) + }) - it("successfully finds multiple rows", () => - expectQuery({ + it("successfully finds multiple rows", async () => { + await expectQuery({ range: { dob: { low: JAN_1ST, high: JAN_10TH } }, - }).toContainExactly([{ dob: JAN_1ST }, { dob: JAN_10TH }])) + }).toContainExactly([{ dob: JAN_1ST }, { dob: JAN_10TH }]) + }) - it("successfully finds a row with a high bound", () => - expectQuery({ + it("successfully finds a row with a high bound", async () => { + await expectQuery({ range: { dob: { low: JAN_5TH, high: JAN_10TH } }, - }).toContainExactly([{ dob: JAN_10TH }])) + }).toContainExactly([{ dob: JAN_10TH }]) + }) - it("successfully finds no rows", () => - expectQuery({ + it("successfully finds no rows", async () => { + await expectQuery({ range: { dob: { low: JAN_5TH, high: JAN_9TH } }, - }).toFindNothing()) + }).toFindNothing() + }) // We never implemented half-open ranges in Lucene. !isLucene && - it("can search using just a low value", () => - expectQuery({ + it("can search using just a low value", async () => { + await expectQuery({ range: { dob: { low: JAN_5TH } }, - }).toContainExactly([{ dob: JAN_10TH }])) + }).toContainExactly([{ dob: JAN_10TH }]) + }) // We never implemented half-open ranges in Lucene. !isLucene && - it("can search using just a high value", () => - expectQuery({ + it("can search using just a high value", async () => { + await expectQuery({ range: { dob: { high: JAN_5TH } }, - }).toContainExactly([{ dob: JAN_1ST }])) + }).toContainExactly([{ dob: JAN_1ST }]) + }) }) describe("sort", () => { - it("sorts ascending", () => - expectSearch({ + it("sorts ascending", async () => { + await expectSearch({ query: {}, sort: "dob", sortOrder: SortOrder.ASCENDING, - }).toMatchExactly([{ dob: JAN_1ST }, { dob: JAN_10TH }])) + }).toMatchExactly([{ dob: JAN_1ST }, { dob: JAN_10TH }]) + }) - it("sorts descending", () => - expectSearch({ + it("sorts descending", async () => { + await expectSearch({ query: {}, sort: "dob", sortOrder: SortOrder.DESCENDING, - }).toMatchExactly([{ dob: JAN_10TH }, { dob: JAN_1ST }])) + }).toMatchExactly([{ dob: JAN_10TH }, { dob: JAN_1ST }]) + }) describe("sortType STRING", () => { - it("sorts ascending", () => - expectSearch({ + it("sorts ascending", async () => { + await expectSearch({ query: {}, sort: "dob", sortType: SortType.STRING, sortOrder: SortOrder.ASCENDING, - }).toMatchExactly([{ dob: JAN_1ST }, { dob: JAN_10TH }])) + }).toMatchExactly([{ dob: JAN_1ST }, { dob: JAN_10TH }]) + }) - it("sorts descending", () => - expectSearch({ + it("sorts descending", async () => { + await expectSearch({ query: {}, sort: "dob", sortType: SortType.STRING, sortOrder: SortOrder.DESCENDING, - }).toMatchExactly([{ dob: JAN_10TH }, { dob: JAN_1ST }])) + }).toMatchExactly([{ dob: JAN_10TH }, { dob: JAN_1ST }]) + }) }) }) }) @@ -1115,72 +1200,85 @@ describe.each([ }) describe("equal", () => { - it("successfully finds a row", () => - expectQuery({ equal: { time: T_1000 } }).toContainExactly([ + it("successfully finds a row", async () => { + await expectQuery({ equal: { time: T_1000 } }).toContainExactly([ { time: "10:00:00" }, - ])) + ]) + }) - it("fails to find nonexistent row", () => - expectQuery({ equal: { time: UNEXISTING_TIME } }).toFindNothing()) + it("fails to find nonexistent row", async () => { + await expectQuery({ + equal: { time: UNEXISTING_TIME }, + }).toFindNothing() + }) }) describe("notEqual", () => { - it("successfully finds a row", () => - expectQuery({ notEqual: { time: T_1000 } }).toContainExactly([ + it("successfully finds a row", async () => { + await expectQuery({ notEqual: { time: T_1000 } }).toContainExactly([ { timeid: NULL_TIME__ID }, { time: "10:45:00" }, { time: "12:00:00" }, { time: "15:30:00" }, { time: "00:00:00" }, - ])) + ]) + }) - it("return all when requesting non-existing", () => - expectQuery({ notEqual: { time: UNEXISTING_TIME } }).toContainExactly( - [ - { timeid: NULL_TIME__ID }, - { time: "10:00:00" }, - { time: "10:45:00" }, - { time: "12:00:00" }, - { time: "15:30:00" }, - { time: "00:00:00" }, - ] - )) + it("return all when requesting non-existing", async () => { + await expectQuery({ + notEqual: { time: UNEXISTING_TIME }, + }).toContainExactly([ + { timeid: NULL_TIME__ID }, + { time: "10:00:00" }, + { time: "10:45:00" }, + { time: "12:00:00" }, + { time: "15:30:00" }, + { time: "00:00:00" }, + ]) + }) }) describe("oneOf", () => { - it("successfully finds a row", () => - expectQuery({ oneOf: { time: [T_1000] } }).toContainExactly([ + it("successfully finds a row", async () => { + await expectQuery({ oneOf: { time: [T_1000] } }).toContainExactly([ { time: "10:00:00" }, - ])) + ]) + }) - it("fails to find nonexistent row", () => - expectQuery({ oneOf: { time: [UNEXISTING_TIME] } }).toFindNothing()) + it("fails to find nonexistent row", async () => { + await expectQuery({ + oneOf: { time: [UNEXISTING_TIME] }, + }).toFindNothing() + }) }) describe("range", () => { - it("successfully finds a row", () => - expectQuery({ + it("successfully finds a row", async () => { + await expectQuery({ range: { time: { low: T_1045, high: T_1045 } }, - }).toContainExactly([{ time: "10:45:00" }])) + }).toContainExactly([{ time: "10:45:00" }]) + }) - it("successfully finds multiple rows", () => - expectQuery({ + it("successfully finds multiple rows", async () => { + await expectQuery({ range: { time: { low: T_1045, high: T_1530 } }, }).toContainExactly([ { time: "10:45:00" }, { time: "12:00:00" }, { time: "15:30:00" }, - ])) + ]) + }) - it("successfully finds no rows", () => - expectQuery({ + it("successfully finds no rows", async () => { + await expectQuery({ range: { time: { low: UNEXISTING_TIME, high: UNEXISTING_TIME } }, - }).toFindNothing()) + }).toFindNothing() + }) }) describe("sort", () => { - it("sorts ascending", () => - expectSearch({ + it("sorts ascending", async () => { + await expectSearch({ query: {}, sort: "time", sortOrder: SortOrder.ASCENDING, @@ -1191,10 +1289,11 @@ describe.each([ { time: "10:45:00" }, { time: "12:00:00" }, { time: "15:30:00" }, - ])) + ]) + }) - it("sorts descending", () => - expectSearch({ + it("sorts descending", async () => { + await expectSearch({ query: {}, sort: "time", sortOrder: SortOrder.DESCENDING, @@ -1205,11 +1304,12 @@ describe.each([ { time: "10:00:00" }, { time: "00:00:00" }, { timeid: NULL_TIME__ID }, - ])) + ]) + }) describe("sortType STRING", () => { - it("sorts ascending", () => - expectSearch({ + it("sorts ascending", async () => { + await expectSearch({ query: {}, sort: "time", sortType: SortType.STRING, @@ -1221,10 +1321,11 @@ describe.each([ { time: "10:45:00" }, { time: "12:00:00" }, { time: "15:30:00" }, - ])) + ]) + }) - it("sorts descending", () => - expectSearch({ + it("sorts descending", async () => { + await expectSearch({ query: {}, sort: "time", sortType: SortType.STRING, @@ -1236,7 +1337,8 @@ describe.each([ { time: "10:00:00" }, { time: "00:00:00" }, { timeid: NULL_TIME__ID }, - ])) + ]) + }) }) }) }) @@ -1254,66 +1356,78 @@ describe.each([ }) describe("contains", () => { - it("successfully finds a row", () => - expectQuery({ contains: { numbers: ["one"] } }).toContainExactly([ + it("successfully finds a row", async () => { + await expectQuery({ contains: { numbers: ["one"] } }).toContainExactly([ { numbers: ["one", "two"] }, - ])) + ]) + }) - it("fails to find nonexistent row", () => - expectQuery({ contains: { numbers: ["none"] } }).toFindNothing()) + it("fails to find nonexistent row", async () => { + await expectQuery({ contains: { numbers: ["none"] } }).toFindNothing() + }) - it("fails to find row containing all", () => - expectQuery({ + it("fails to find row containing all", async () => { + await expectQuery({ contains: { numbers: ["one", "two", "three"] }, - }).toFindNothing()) + }).toFindNothing() + }) - it("finds all with empty list", () => - expectQuery({ contains: { numbers: [] } }).toContainExactly([ + it("finds all with empty list", async () => { + await expectQuery({ contains: { numbers: [] } }).toContainExactly([ { numbers: ["one", "two"] }, { numbers: ["three"] }, - ])) + ]) + }) }) describe("notContains", () => { - it("successfully finds a row", () => - expectQuery({ notContains: { numbers: ["one"] } }).toContainExactly([ - { numbers: ["three"] }, - ])) + it("successfully finds a row", async () => { + await expectQuery({ + notContains: { numbers: ["one"] }, + }).toContainExactly([{ numbers: ["three"] }]) + }) - it("fails to find nonexistent row", () => - expectQuery({ + it("fails to find nonexistent row", async () => { + await expectQuery({ notContains: { numbers: ["one", "two", "three"] }, }).toContainExactly([ { numbers: ["one", "two"] }, { numbers: ["three"] }, - ])) + ]) + }) // Not sure if this is correct behaviour but changing it would be a // breaking change. - it("finds all with empty list", () => - expectQuery({ notContains: { numbers: [] } }).toContainExactly([ + it("finds all with empty list", async () => { + await expectQuery({ notContains: { numbers: [] } }).toContainExactly([ { numbers: ["one", "two"] }, { numbers: ["three"] }, - ])) + ]) + }) }) describe("containsAny", () => { - it("successfully finds rows", () => - expectQuery({ + it("successfully finds rows", async () => { + await expectQuery({ containsAny: { numbers: ["one", "two", "three"] }, }).toContainExactly([ { numbers: ["one", "two"] }, { numbers: ["three"] }, - ])) + ]) + }) - it("fails to find nonexistent row", () => - expectQuery({ containsAny: { numbers: ["none"] } }).toFindNothing()) + it("fails to find nonexistent row", async () => { + await expectQuery({ + containsAny: { numbers: ["none"] }, + }).toFindNothing() + }) - it("finds all with empty list", () => - expectQuery({ containsAny: { numbers: [] } }).toContainExactly([ + it("finds all with empty list", async () => { + await expectQuery({ containsAny: { numbers: [] } }).toContainExactly([ { numbers: ["one", "two"] }, { numbers: ["three"] }, - ])) + ]) + }) }) }) @@ -1332,48 +1446,56 @@ describe.each([ }) describe("equal", () => { - it("successfully finds a row", () => - expectQuery({ equal: { num: SMALL } }).toContainExactly([ + it("successfully finds a row", async () => { + await expectQuery({ equal: { num: SMALL } }).toContainExactly([ { num: SMALL }, - ])) + ]) + }) - it("successfully finds a big value", () => - expectQuery({ equal: { num: BIG } }).toContainExactly([{ num: BIG }])) + it("successfully finds a big value", async () => { + await expectQuery({ equal: { num: BIG } }).toContainExactly([ + { num: BIG }, + ]) + }) - it("fails to find nonexistent row", () => - expectQuery({ equal: { num: "2" } }).toFindNothing()) + it("fails to find nonexistent row", async () => { + await expectQuery({ equal: { num: "2" } }).toFindNothing() + }) }) describe("notEqual", () => { - it("successfully finds a row", () => - expectQuery({ notEqual: { num: SMALL } }).toContainExactly([ + it("successfully finds a row", async () => { + await expectQuery({ notEqual: { num: SMALL } }).toContainExactly([ { num: MEDIUM }, { num: BIG }, - ])) + ]) + }) - it("fails to find nonexistent row", () => - expectQuery({ notEqual: { num: 10 } }).toContainExactly([ + it("fails to find nonexistent row", async () => { + await expectQuery({ notEqual: { num: 10 } }).toContainExactly([ { num: SMALL }, { num: MEDIUM }, { num: BIG }, - ])) + ]) + }) }) describe("oneOf", () => { - it("successfully finds a row", () => - expectQuery({ oneOf: { num: [SMALL] } }).toContainExactly([ + it("successfully finds a row", async () => { + await expectQuery({ oneOf: { num: [SMALL] } }).toContainExactly([ { num: SMALL }, - ])) + ]) + }) - it("successfully finds all rows", () => - expectQuery({ oneOf: { num: [SMALL, MEDIUM, BIG] } }).toContainExactly([ - { num: SMALL }, - { num: MEDIUM }, - { num: BIG }, - ])) + it("successfully finds all rows", async () => { + await expectQuery({ + oneOf: { num: [SMALL, MEDIUM, BIG] }, + }).toContainExactly([{ num: SMALL }, { num: MEDIUM }, { num: BIG }]) + }) - it("fails to find nonexistent row", () => - expectQuery({ oneOf: { num: [2] } }).toFindNothing()) + it("fails to find nonexistent row", async () => { + await expectQuery({ oneOf: { num: [2] } }).toFindNothing() + }) }) // Range searches against bigints don't seem to work at all in Lucene, and I @@ -1381,35 +1503,41 @@ describe.each([ // we've decided not to spend time on it. !isLucene && describe("range", () => { - it("successfully finds a row", () => - expectQuery({ + it("successfully finds a row", async () => { + await expectQuery({ range: { num: { low: SMALL, high: "5" } }, - }).toContainExactly([{ num: SMALL }])) + }).toContainExactly([{ num: SMALL }]) + }) - it("successfully finds multiple rows", () => - expectQuery({ + it("successfully finds multiple rows", async () => { + await expectQuery({ range: { num: { low: SMALL, high: MEDIUM } }, - }).toContainExactly([{ num: SMALL }, { num: MEDIUM }])) + }).toContainExactly([{ num: SMALL }, { num: MEDIUM }]) + }) - it("successfully finds a row with a high bound", () => - expectQuery({ + it("successfully finds a row with a high bound", async () => { + await expectQuery({ range: { num: { low: MEDIUM, high: BIG } }, - }).toContainExactly([{ num: MEDIUM }, { num: BIG }])) + }).toContainExactly([{ num: MEDIUM }, { num: BIG }]) + }) - it("successfully finds no rows", () => - expectQuery({ + it("successfully finds no rows", async () => { + await expectQuery({ range: { num: { low: "5", high: "5" } }, - }).toFindNothing()) + }).toFindNothing() + }) - it("can search using just a low value", () => - expectQuery({ + it("can search using just a low value", async () => { + await expectQuery({ range: { num: { low: MEDIUM } }, - }).toContainExactly([{ num: MEDIUM }, { num: BIG }])) + }).toContainExactly([{ num: MEDIUM }, { num: BIG }]) + }) - it("can search using just a high value", () => - expectQuery({ + it("can search using just a high value", async () => { + await expectQuery({ range: { num: { high: MEDIUM } }, - }).toContainExactly([{ num: SMALL }, { num: MEDIUM }])) + }).toContainExactly([{ num: SMALL }, { num: MEDIUM }]) + }) }) }) @@ -1428,16 +1556,20 @@ describe.each([ }) describe("equal", () => { - it("successfully finds a row", () => - expectQuery({ equal: { auto: 1 } }).toContainExactly([{ auto: 1 }])) + it("successfully finds a row", async () => { + await expectQuery({ equal: { auto: 1 } }).toContainExactly([ + { auto: 1 }, + ]) + }) - it("fails to find nonexistent row", () => - expectQuery({ equal: { auto: 0 } }).toFindNothing()) + it("fails to find nonexistent row", async () => { + await expectQuery({ equal: { auto: 0 } }).toFindNothing() + }) }) describe("not equal", () => { - it("successfully finds a row", () => - expectQuery({ notEqual: { auto: 1 } }).toContainExactly([ + it("successfully finds a row", async () => { + await expectQuery({ notEqual: { auto: 1 } }).toContainExactly([ { auto: 2 }, { auto: 3 }, { auto: 4 }, @@ -1447,10 +1579,11 @@ describe.each([ { auto: 8 }, { auto: 9 }, { auto: 10 }, - ])) + ]) + }) - it("fails to find nonexistent row", () => - expectQuery({ notEqual: { auto: 0 } }).toContainExactly([ + it("fails to find nonexistent row", async () => { + await expectQuery({ notEqual: { auto: 0 } }).toContainExactly([ { auto: 1 }, { auto: 2 }, { auto: 3 }, @@ -1461,55 +1594,66 @@ describe.each([ { auto: 8 }, { auto: 9 }, { auto: 10 }, - ])) + ]) + }) }) describe("oneOf", () => { - it("successfully finds a row", () => - expectQuery({ oneOf: { auto: [1] } }).toContainExactly([{ auto: 1 }])) + it("successfully finds a row", async () => { + await expectQuery({ oneOf: { auto: [1] } }).toContainExactly([ + { auto: 1 }, + ]) + }) - it("fails to find nonexistent row", () => - expectQuery({ oneOf: { auto: [0] } }).toFindNothing()) + it("fails to find nonexistent row", async () => { + await expectQuery({ oneOf: { auto: [0] } }).toFindNothing() + }) }) describe("range", () => { - it("successfully finds a row", () => - expectQuery({ + it("successfully finds a row", async () => { + await expectQuery({ range: { auto: { low: 1, high: 1 } }, - }).toContainExactly([{ auto: 1 }])) + }).toContainExactly([{ auto: 1 }]) + }) - it("successfully finds multiple rows", () => - expectQuery({ + it("successfully finds multiple rows", async () => { + await expectQuery({ range: { auto: { low: 1, high: 2 } }, - }).toContainExactly([{ auto: 1 }, { auto: 2 }])) + }).toContainExactly([{ auto: 1 }, { auto: 2 }]) + }) - it("successfully finds a row with a high bound", () => - expectQuery({ + it("successfully finds a row with a high bound", async () => { + await expectQuery({ range: { auto: { low: 2, high: 2 } }, - }).toContainExactly([{ auto: 2 }])) + }).toContainExactly([{ auto: 2 }]) + }) - it("successfully finds no rows", () => - expectQuery({ + it("successfully finds no rows", async () => { + await expectQuery({ range: { auto: { low: 0, high: 0 } }, - }).toFindNothing()) + }).toFindNothing() + }) isSqs && - it("can search using just a low value", () => - expectQuery({ + it("can search using just a low value", async () => { + await expectQuery({ range: { auto: { low: 9 } }, - }).toContainExactly([{ auto: 9 }, { auto: 10 }])) + }).toContainExactly([{ auto: 9 }, { auto: 10 }]) + }) isSqs && - it("can search using just a high value", () => - expectQuery({ + it("can search using just a high value", async () => { + await expectQuery({ range: { auto: { high: 2 } }, - }).toContainExactly([{ auto: 1 }, { auto: 2 }])) + }).toContainExactly([{ auto: 1 }, { auto: 2 }]) + }) }) isSqs && describe("sort", () => { - it("sorts ascending", () => - expectSearch({ + it("sorts ascending", async () => { + await expectSearch({ query: {}, sort: "auto", sortOrder: SortOrder.ASCENDING, @@ -1524,10 +1668,11 @@ describe.each([ { auto: 8 }, { auto: 9 }, { auto: 10 }, - ])) + ]) + }) - it("sorts descending", () => - expectSearch({ + it("sorts descending", async () => { + await expectSearch({ query: {}, sort: "auto", sortOrder: SortOrder.DESCENDING, @@ -1542,25 +1687,39 @@ describe.each([ { auto: 3 }, { auto: 2 }, { auto: 1 }, - ])) + ]) + }) // This is important for pagination. The order of results must always // be stable or pagination will break. We don't want the user to need // to specify an order for pagination to work. it("is stable without a sort specified", async () => { - let { rows } = await config.api.row.search(table._id!, { - tableId: table._id!, - query: {}, - }) + let { rows: fullRowList } = await config.api.row.search( + table._id!, + { + tableId: table._id!, + query: {}, + } + ) - for (let i = 0; i < 10; i++) { + // repeat the search many times to check the first row is always the same + let bookmark: string | number | undefined, + hasNextPage: boolean | undefined = true, + rowCount: number = 0 + do { const response = await config.api.row.search(table._id!, { tableId: table._id!, limit: 1, + paginate: true, query: {}, + bookmark, }) - expect(response.rows).toEqual(rows) - } + bookmark = response.bookmark + hasNextPage = response.hasNextPage + expect(response.rows.length).toEqual(1) + const foundRow = response.rows[0] + expect(foundRow).toEqual(fullRowList[rowCount++]) + } while (hasNextPage) }) }) @@ -1602,13 +1761,15 @@ describe.each([ await createRows([{ "1:name": "bar" }, { "1:name": "foo" }]) }) - it("successfully finds a row", () => - expectQuery({ equal: { "1:1:name": "bar" } }).toContainExactly([ + it("successfully finds a row", async () => { + await expectQuery({ equal: { "1:1:name": "bar" } }).toContainExactly([ { "1:name": "bar" }, - ])) + ]) + }) - it("fails to find nonexistent row", () => - expectQuery({ equal: { "1:1:name": "none" } }).toFindNothing()) + it("fails to find nonexistent row", async () => { + await expectQuery({ equal: { "1:1:name": "none" } }).toFindNothing() + }) }) describe("user", () => { @@ -1635,51 +1796,59 @@ describe.each([ }) describe("equal", () => { - it("successfully finds a row", () => - expectQuery({ equal: { user: user1._id } }).toContainExactly([ + it("successfully finds a row", async () => { + await expectQuery({ equal: { user: user1._id } }).toContainExactly([ { user: { _id: user1._id } }, - ])) + ]) + }) - it("fails to find nonexistent row", () => - expectQuery({ equal: { user: "us_none" } }).toFindNothing()) + it("fails to find nonexistent row", async () => { + await expectQuery({ equal: { user: "us_none" } }).toFindNothing() + }) }) describe("notEqual", () => { - it("successfully finds a row", () => - expectQuery({ notEqual: { user: user1._id } }).toContainExactly([ + it("successfully finds a row", async () => { + await expectQuery({ notEqual: { user: user1._id } }).toContainExactly([ { user: { _id: user2._id } }, {}, - ])) + ]) + }) - it("fails to find nonexistent row", () => - expectQuery({ notEqual: { user: "us_none" } }).toContainExactly([ + it("fails to find nonexistent row", async () => { + await expectQuery({ notEqual: { user: "us_none" } }).toContainExactly([ { user: { _id: user1._id } }, { user: { _id: user2._id } }, {}, - ])) + ]) + }) }) describe("oneOf", () => { - it("successfully finds a row", () => - expectQuery({ oneOf: { user: [user1._id] } }).toContainExactly([ + it("successfully finds a row", async () => { + await expectQuery({ oneOf: { user: [user1._id] } }).toContainExactly([ { user: { _id: user1._id } }, - ])) + ]) + }) - it("fails to find nonexistent row", () => - expectQuery({ oneOf: { user: ["us_none"] } }).toFindNothing()) + it("fails to find nonexistent row", async () => { + await expectQuery({ oneOf: { user: ["us_none"] } }).toFindNothing() + }) }) describe("empty", () => { - it("finds empty rows", () => - expectQuery({ empty: { user: null } }).toContainExactly([{}])) + it("finds empty rows", async () => { + await expectQuery({ empty: { user: null } }).toContainExactly([{}]) + }) }) describe("notEmpty", () => { - it("finds non-empty rows", () => - expectQuery({ notEmpty: { user: null } }).toContainExactly([ + it("finds non-empty rows", async () => { + await expectQuery({ notEmpty: { user: null } }).toContainExactly([ { user: { _id: user1._id } }, { user: { _id: user2._id } }, - ])) + ]) + }) }) }) @@ -1713,58 +1882,71 @@ describe.each([ }) describe("contains", () => { - it("successfully finds a row", () => - expectQuery({ contains: { users: [user1._id] } }).toContainExactly([ + it("successfully finds a row", async () => { + await expectQuery({ + contains: { users: [user1._id] }, + }).toContainExactly([ { users: [{ _id: user1._id }] }, { users: [{ _id: user1._id }, { _id: user2._id }] }, - ])) + ]) + }) - it("fails to find nonexistent row", () => - expectQuery({ contains: { users: ["us_none"] } }).toFindNothing()) + it("fails to find nonexistent row", async () => { + await expectQuery({ contains: { users: ["us_none"] } }).toFindNothing() + }) }) describe("notContains", () => { - it("successfully finds a row", () => - expectQuery({ notContains: { users: [user1._id] } }).toContainExactly([ - { users: [{ _id: user2._id }] }, - {}, - ])) + it("successfully finds a row", async () => { + await expectQuery({ + notContains: { users: [user1._id] }, + }).toContainExactly([{ users: [{ _id: user2._id }] }, {}]) + }) - it("fails to find nonexistent row", () => - expectQuery({ notContains: { users: ["us_none"] } }).toContainExactly([ + it("fails to find nonexistent row", async () => { + await expectQuery({ + notContains: { users: ["us_none"] }, + }).toContainExactly([ { users: [{ _id: user1._id }] }, { users: [{ _id: user2._id }] }, { users: [{ _id: user1._id }, { _id: user2._id }] }, {}, - ])) + ]) + }) }) describe("containsAny", () => { - it("successfully finds rows", () => - expectQuery({ + it("successfully finds rows", async () => { + await expectQuery({ containsAny: { users: [user1._id, user2._id] }, }).toContainExactly([ { users: [{ _id: user1._id }] }, { users: [{ _id: user2._id }] }, { users: [{ _id: user1._id }, { _id: user2._id }] }, - ])) + ]) + }) - it("fails to find nonexistent row", () => - expectQuery({ containsAny: { users: ["us_none"] } }).toFindNothing()) + it("fails to find nonexistent row", async () => { + await expectQuery({ + containsAny: { users: ["us_none"] }, + }).toFindNothing() + }) }) describe("multi-column equals", () => { - it("successfully finds a row", () => - expectQuery({ + it("successfully finds a row", async () => { + await expectQuery({ equal: { number: 1 }, contains: { users: [user1._id] }, - }).toContainExactly([{ users: [{ _id: user1._id }], number: 1 }])) + }).toContainExactly([{ users: [{ _id: user1._id }], number: 1 }]) + }) - it("fails to find nonexistent row", () => - expectQuery({ + it("fails to find nonexistent row", async () => { + await expectQuery({ equal: { number: 2 }, contains: { users: [user1._id] }, - }).toFindNothing()) + }).toFindNothing() + }) }) }) @@ -1774,52 +1956,74 @@ describe.each([ // isn't available. !isInMemory && describe("relations", () => { - let otherTable: Table - let otherRows: Row[] + let productCategoryTable: Table, productCatRows: Row[] beforeAll(async () => { - otherTable = await createTable({ - one: { name: "one", type: FieldType.STRING }, - }) - table = await createTable({ - two: { name: "two", type: FieldType.STRING }, - other: { - type: FieldType.LINK, - relationshipType: RelationshipType.ONE_TO_MANY, - name: "other", - fieldName: "other", - tableId: otherTable._id!, - constraints: { - type: "array", + productCategoryTable = await createTable( + { + name: { name: "name", type: FieldType.STRING }, + }, + "productCategory" + ) + table = await createTable( + { + name: { name: "name", type: FieldType.STRING }, + productCat: { + type: FieldType.LINK, + relationshipType: RelationshipType.ONE_TO_MANY, + name: "productCat", + fieldName: "product", + tableId: productCategoryTable._id!, + constraints: { + type: "array", + }, }, }, - }) + "product" + ) - otherRows = await Promise.all([ - config.api.row.save(otherTable._id!, { one: "foo" }), - config.api.row.save(otherTable._id!, { one: "bar" }), + productCatRows = await Promise.all([ + config.api.row.save(productCategoryTable._id!, { name: "foo" }), + config.api.row.save(productCategoryTable._id!, { name: "bar" }), ]) await Promise.all([ config.api.row.save(table._id!, { - two: "foo", - other: [otherRows[0]._id], + name: "foo", + productCat: [productCatRows[0]._id], }), config.api.row.save(table._id!, { - two: "bar", - other: [otherRows[1]._id], + name: "bar", + productCat: [productCatRows[1]._id], + }), + config.api.row.save(table._id!, { + name: "baz", + productCat: [], }), ]) - - rows = await config.api.row.fetch(table._id!) }) - it("can search through relations", () => - expectQuery({ - equal: { [`${otherTable.name}.one`]: "foo" }, + it("should be able to filter by relationship using column name", async () => { + await expectQuery({ + equal: { ["productCat.name"]: "foo" }, }).toContainExactly([ - { two: "foo", other: [{ _id: otherRows[0]._id }] }, - ])) + { name: "foo", productCat: [{ _id: productCatRows[0]._id }] }, + ]) + }) + + it("should be able to filter by relationship using table name", async () => { + await expectQuery({ + equal: { ["productCategory.name"]: "foo" }, + }).toContainExactly([ + { name: "foo", productCat: [{ _id: productCatRows[0]._id }] }, + ]) + }) + + it("shouldn't return any relationship for last row", async () => { + await expectQuery({ + equal: { ["name"]: "baz" }, + }).toContainExactly([{ name: "baz", productCat: undefined }]) + }) }) // lucene can't count the total rows @@ -1835,18 +2039,19 @@ describe.each([ await createRows([{ name: "a" }, { name: "b" }]) }) - it("should be able to count rows when option set", () => - expectSearch({ + it("should be able to count rows when option set", async () => { + await expectSearch({ countRows: true, query: { notEmpty: { name: true, }, }, - }).toMatch({ totalRows: 2, rows: expect.any(Array) })) + }).toMatch({ totalRows: 2, rows: expect.any(Array) }) + }) - it("shouldn't count rows when option is not set", () => { - expectSearch({ + it("shouldn't count rows when option is not set", async () => { + await expectSearch({ countRows: false, query: { notEmpty: { diff --git a/packages/server/src/api/routes/tests/table.spec.ts b/packages/server/src/api/routes/tests/table.spec.ts index f23e0de6db..e75e5e23e7 100644 --- a/packages/server/src/api/routes/tests/table.spec.ts +++ b/packages/server/src/api/routes/tests/table.spec.ts @@ -276,6 +276,34 @@ describe.each([ }) }) + isInternal && + it("shouldn't allow duplicate column names", async () => { + const saveTableRequest: SaveTableRequest = { + ...basicTable(), + } + saveTableRequest.schema["Type"] = { + type: FieldType.STRING, + name: "Type", + } + await config.api.table.save(saveTableRequest, { + status: 400, + body: { + message: + 'Column(s) "type" are duplicated - check for other columns with these name (case in-sensitive)', + }, + }) + saveTableRequest.schema.foo = { type: FieldType.STRING, name: "foo" } + saveTableRequest.schema.FOO = { type: FieldType.STRING, name: "FOO" } + + await config.api.table.save(saveTableRequest, { + status: 400, + body: { + message: + 'Column(s) "type, foo" are duplicated - check for other columns with these name (case in-sensitive)', + }, + }) + }) + it("should add a new column for an internal DB table", async () => { const saveTableRequest: SaveTableRequest = { ...basicTable(), diff --git a/packages/server/src/sdk/app/rows/index.ts b/packages/server/src/sdk/app/rows/index.ts index c117941419..fb077509a9 100644 --- a/packages/server/src/sdk/app/rows/index.ts +++ b/packages/server/src/sdk/app/rows/index.ts @@ -3,12 +3,14 @@ import * as rows from "./rows" import * as search from "./search" import * as utils from "./utils" import * as external from "./external" +import * as filters from "./search/filters" import AliasTables from "./sqlAlias" export default { ...attachments, ...rows, ...search, + filters, utils, external, AliasTables, diff --git a/packages/server/src/sdk/app/rows/search/external.ts b/packages/server/src/sdk/app/rows/search/external.ts index 9fc3487f62..93c46d8cc3 100644 --- a/packages/server/src/sdk/app/rows/search/external.ts +++ b/packages/server/src/sdk/app/rows/search/external.ts @@ -145,6 +145,10 @@ export async function exportRows( delimiter, customHeaders, } = options + + if (!tableId) { + throw new HTTPError("No table ID for search provided.", 400) + } const { datasourceId, tableName } = breakExternalTableId(tableId) let requestQuery: SearchFilters = {} @@ -167,7 +171,7 @@ export async function exportRows( requestQuery = query || {} } - const datasource = await sdk.datasources.get(datasourceId!) + const datasource = await sdk.datasources.get(datasourceId) const table = await sdk.tables.getTable(tableId) if (!datasource || !datasource.entities) { throw new HTTPError("Datasource has not been configured for plus API.", 400) @@ -180,10 +184,6 @@ export async function exportRows( let rows: Row[] = [] let headers - if (!tableName) { - throw new HTTPError("Could not find table name.", 400) - } - // Filter data to only specified columns if required if (columns && columns.length) { for (let i = 0; i < result.rows.length; i++) { diff --git a/packages/server/src/sdk/app/rows/search/filters.ts b/packages/server/src/sdk/app/rows/search/filters.ts new file mode 100644 index 0000000000..ccce0ab86a --- /dev/null +++ b/packages/server/src/sdk/app/rows/search/filters.ts @@ -0,0 +1,62 @@ +import { + FieldType, + RelationshipFieldMetadata, + SearchFilters, + Table, +} from "@budibase/types" +import { isPlainObject } from "lodash" + +export function getRelationshipColumns(table: Table): { + name: string + definition: RelationshipFieldMetadata +}[] { + // performing this with a for loop rather than an array filter improves + // type guarding, as no casts are required + const linkEntries: [string, RelationshipFieldMetadata][] = [] + for (let entry of Object.entries(table.schema)) { + if (entry[1].type === FieldType.LINK) { + const linkColumn: RelationshipFieldMetadata = entry[1] + linkEntries.push([entry[0], linkColumn]) + } + } + return linkEntries.map(entry => ({ + name: entry[0], + definition: entry[1], + })) +} + +export function getTableIDList( + tables: Table[] +): { name: string; id: string }[] { + return tables + .filter(table => table.originalName && table._id) + .map(table => ({ id: table._id!, name: table.originalName! })) +} + +export function updateFilterKeys( + filters: SearchFilters, + updates: { original: string; updated: string }[] +): SearchFilters { + const makeFilterKeyRegex = (str: string) => + new RegExp(`^${str}\\.|:${str}\\.`) + for (let filter of Object.values(filters)) { + if (!isPlainObject(filter)) { + continue + } + for (let [key, keyFilter] of Object.entries(filter)) { + if (keyFilter === "") { + delete filter[key] + } + const possibleKey = updates.find(({ original }) => + key.match(makeFilterKeyRegex(original)) + ) + if (possibleKey && possibleKey.original !== possibleKey.updated) { + // only replace the first, not replaceAll + filter[key.replace(possibleKey.original, possibleKey.updated)] = + filter[key] + delete filter[key] + } + } + } + return filters +} diff --git a/packages/server/src/sdk/app/rows/search/sqs.ts b/packages/server/src/sdk/app/rows/search/sqs.ts index bb1d62affc..174ecc0e38 100644 --- a/packages/server/src/sdk/app/rows/search/sqs.ts +++ b/packages/server/src/sdk/app/rows/search/sqs.ts @@ -1,4 +1,5 @@ import { + Datasource, DocumentType, FieldType, Operation, @@ -12,7 +13,6 @@ import { SortType, SqlClient, Table, - Datasource, } from "@budibase/types" import { buildInternalRelationships, @@ -30,6 +30,11 @@ import AliasTables from "../sqlAlias" import { outputProcessing } from "../../../../utilities/rowProcessor" import pick from "lodash/pick" import { processRowCountResponse } from "../utils" +import { + updateFilterKeys, + getRelationshipColumns, + getTableIDList, +} from "./filters" const builder = new sql.Sql(SqlClient.SQL_LITE) @@ -60,34 +65,31 @@ function buildInternalFieldList( return fieldList } -function tableNameInFieldRegex(tableName: string) { - return new RegExp(`^${tableName}.|:${tableName}.`, "g") -} - -function cleanupFilters(filters: SearchFilters, tables: Table[]) { - for (let filter of Object.values(filters)) { - if (typeof filter !== "object") { - continue - } - for (let [key, keyFilter] of Object.entries(filter)) { - if (keyFilter === "") { - delete filter[key] - } - - // relationship, switch to table ID - const tableRelated = tables.find( - table => - table.originalName && - key.match(tableNameInFieldRegex(table.originalName)) +function cleanupFilters( + filters: SearchFilters, + table: Table, + allTables: Table[] +) { + // get a list of all relationship columns in the table for updating + const relationshipColumns = getRelationshipColumns(table) + // get table names to ID map for relationships + const tableNameToID = getTableIDList(allTables) + // all should be applied at once + filters = updateFilterKeys( + filters, + relationshipColumns + .map(({ name, definition }) => ({ + original: name, + updated: definition.tableId, + })) + .concat( + tableNameToID.map(({ name, id }) => ({ + original: name, + updated: id, + })) ) - if (tableRelated && tableRelated.originalName) { - // only replace the first, not replaceAll - filter[key.replace(tableRelated.originalName, tableRelated._id!)] = - filter[key] - delete filter[key] - } - } - } + ) + return filters } @@ -176,7 +178,7 @@ export async function search( operation: Operation.READ, }, filters: { - ...cleanupFilters(query, allTables), + ...cleanupFilters(query, table, allTables), documentType: DocumentType.ROW, }, table, diff --git a/packages/server/src/sdk/app/tables/getters.ts b/packages/server/src/sdk/app/tables/getters.ts index 355493579d..738e57eff8 100644 --- a/packages/server/src/sdk/app/tables/getters.ts +++ b/packages/server/src/sdk/app/tables/getters.ts @@ -90,10 +90,10 @@ export async function getExternalTable( export async function getTable(tableId: string): Promise { const db = context.getAppDB() let output: Table - if (isExternalTableID(tableId)) { + if (tableId && isExternalTableID(tableId)) { let { datasourceId, tableName } = breakExternalTableId(tableId) - const datasource = await datasources.get(datasourceId!) - const table = await getExternalTable(datasourceId!, tableName!) + const datasource = await datasources.get(datasourceId) + const table = await getExternalTable(datasourceId, tableName) output = { ...table, sql: isSQL(datasource) } } else { output = await db.get
(tableId) diff --git a/packages/server/src/sdk/app/tables/internal/index.ts b/packages/server/src/sdk/app/tables/internal/index.ts index ea40d2bfe9..fc32708708 100644 --- a/packages/server/src/sdk/app/tables/internal/index.ts +++ b/packages/server/src/sdk/app/tables/internal/index.ts @@ -17,6 +17,7 @@ import { cloneDeep } from "lodash/fp" import isEqual from "lodash/isEqual" import { runStaticFormulaChecks } from "../../../../api/controllers/table/bulkFormula" import { context } from "@budibase/backend-core" +import { findDuplicateInternalColumns } from "@budibase/shared-core" import { getTable } from "../getters" import { checkAutoColumns } from "./utils" import * as viewsSdk from "../../views" @@ -44,6 +45,17 @@ export async function save( if (hasTypeChanged(table, oldTable)) { throw new Error("A column type has changed.") } + + // check for case sensitivity - we don't want to allow duplicated columns + const duplicateColumn = findDuplicateInternalColumns(table) + if (duplicateColumn.length) { + throw new Error( + `Column(s) "${duplicateColumn.join( + ", " + )}" are duplicated - check for other columns with these name (case in-sensitive)` + ) + } + // check that subtypes have been maintained table = checkAutoColumns(table, oldTable) diff --git a/packages/server/src/sdk/app/views/external.ts b/packages/server/src/sdk/app/views/external.ts index 0f96bcc061..2b3e271597 100644 --- a/packages/server/src/sdk/app/views/external.ts +++ b/packages/server/src/sdk/app/views/external.ts @@ -10,9 +10,9 @@ export async function get(viewId: string): Promise { const { tableId } = utils.extractViewInfoFromID(viewId) const { datasourceId, tableName } = breakExternalTableId(tableId) - const ds = await sdk.datasources.get(datasourceId!) + const ds = await sdk.datasources.get(datasourceId) - const table = ds.entities![tableName!] + const table = ds.entities![tableName] const views = Object.values(table.views!).filter(isV2) const found = views.find(v => v.id === viewId) if (!found) { @@ -25,9 +25,9 @@ export async function getEnriched(viewId: string): Promise { const { tableId } = utils.extractViewInfoFromID(viewId) const { datasourceId, tableName } = breakExternalTableId(tableId) - const ds = await sdk.datasources.get(datasourceId!) + const ds = await sdk.datasources.get(datasourceId) - const table = ds.entities![tableName!] + const table = ds.entities![tableName] const views = Object.values(table.views!).filter(isV2) const found = views.find(v => v.id === viewId) if (!found) { @@ -49,9 +49,9 @@ export async function create( const db = context.getAppDB() const { datasourceId, tableName } = breakExternalTableId(tableId) - const ds = await sdk.datasources.get(datasourceId!) - ds.entities![tableName!].views ??= {} - ds.entities![tableName!].views![view.name] = view + const ds = await sdk.datasources.get(datasourceId) + ds.entities![tableName].views ??= {} + ds.entities![tableName].views![view.name] = view await db.put(ds) return view } @@ -60,9 +60,9 @@ export async function update(tableId: string, view: ViewV2): Promise { const db = context.getAppDB() const { datasourceId, tableName } = breakExternalTableId(tableId) - const ds = await sdk.datasources.get(datasourceId!) - ds.entities![tableName!].views ??= {} - const views = ds.entities![tableName!].views! + const ds = await sdk.datasources.get(datasourceId) + ds.entities![tableName].views ??= {} + const views = ds.entities![tableName].views! const existingView = Object.values(views).find( v => isV2(v) && v.id === view.id @@ -87,9 +87,9 @@ export async function remove(viewId: string): Promise { } const { datasourceId, tableName } = breakExternalTableId(view.tableId) - const ds = await sdk.datasources.get(datasourceId!) + const ds = await sdk.datasources.get(datasourceId) - delete ds.entities![tableName!].views![view?.name] + delete ds.entities![tableName].views![view?.name] await db.put(ds) return view } diff --git a/packages/shared-core/src/constants/index.ts b/packages/shared-core/src/constants/index.ts index afb7e659e1..c9d1a8fc8f 100644 --- a/packages/shared-core/src/constants/index.ts +++ b/packages/shared-core/src/constants/index.ts @@ -1,5 +1,6 @@ export * from "./api" export * from "./fields" +export * from "./rows" export const OperatorOptions = { Equals: { @@ -179,3 +180,5 @@ export enum BpmStatusValue { VERIFYING_EMAIL = "verifying_email", COMPLETED = "completed", } + +export const DEFAULT_BB_DATASOURCE_ID = "datasource_internal_bb_default" diff --git a/packages/shared-core/src/constants/rows.ts b/packages/shared-core/src/constants/rows.ts new file mode 100644 index 0000000000..bfa7595d62 --- /dev/null +++ b/packages/shared-core/src/constants/rows.ts @@ -0,0 +1,14 @@ +export const CONSTANT_INTERNAL_ROW_COLS = [ + "_id", + "_rev", + "type", + "createdAt", + "updatedAt", + "tableId", +] as const + +export const CONSTANT_EXTERNAL_ROW_COLS = ["_id", "_rev", "tableId"] as const + +export function isInternalColumnName(name: string): boolean { + return (CONSTANT_INTERNAL_ROW_COLS as readonly string[]).includes(name) +} diff --git a/packages/shared-core/src/table.ts b/packages/shared-core/src/table.ts index 7706b78037..4b578a2aef 100644 --- a/packages/shared-core/src/table.ts +++ b/packages/shared-core/src/table.ts @@ -1,4 +1,5 @@ -import { FieldType } from "@budibase/types" +import { FieldType, Table } from "@budibase/types" +import { CONSTANT_INTERNAL_ROW_COLS } from "./constants" const allowDisplayColumnByType: Record = { [FieldType.STRING]: true, @@ -51,3 +52,22 @@ export function canBeDisplayColumn(type: FieldType): boolean { export function canBeSortColumn(type: FieldType): boolean { return !!allowSortColumnByType[type] } + +export function findDuplicateInternalColumns(table: Table): string[] { + // get the column names + const columnNames = Object.keys(table.schema) + .concat(CONSTANT_INTERNAL_ROW_COLS) + .map(colName => colName.toLowerCase()) + // there are duplicates + const set = new Set(columnNames) + let duplicates: string[] = [] + if (set.size !== columnNames.length) { + for (let key of set.keys()) { + const count = columnNames.filter(name => name === key).length + if (count > 1) { + duplicates.push(key) + } + } + } + return duplicates +} diff --git a/yarn.lock b/yarn.lock index 3606f068b1..9914c334df 100644 --- a/yarn.lock +++ b/yarn.lock @@ -10296,7 +10296,7 @@ engine.io-parser@~5.0.3: resolved "https://registry.yarnpkg.com/engine.io-parser/-/engine.io-parser-5.0.6.tgz#7811244af173e157295dec9b2718dfe42a64ef45" integrity sha512-tjuoZDMAdEhVnSFleYPCtdL2GXwVTGtNjoeJd9IhIG3C1xs9uwxqRNEu5WpnDZCaozwVlK/nuQhpodhXSIMaxw== -engine.io@~6.4.1: +engine.io@~6.4.2: version "6.4.2" resolved "https://registry.yarnpkg.com/engine.io/-/engine.io-6.4.2.tgz#ffeaf68f69b1364b0286badddf15ff633476473f" integrity sha512-FKn/3oMiJjrOEOeUub2WCox6JhxBXq/Zn3fZOMCBxKnNYtsdKjxhl7yR3fZhM9PV+rdE75SU5SYMc+2PGzo+Tg== @@ -20160,17 +20160,25 @@ socket.io-parser@~4.2.1: "@socket.io/component-emitter" "~3.1.0" debug "~4.3.1" -socket.io@4.6.1: - version "4.6.1" - resolved "https://registry.yarnpkg.com/socket.io/-/socket.io-4.6.1.tgz#62ec117e5fce0692fa50498da9347cfb52c3bc70" - integrity sha512-KMcaAi4l/8+xEjkRICl6ak8ySoxsYG+gG6/XfRCPJPQ/haCRIJBTL4wIl8YCsmtaBovcAXGLOShyVWQ/FG8GZA== +socket.io-parser@~4.2.4: + version "4.2.4" + resolved "https://registry.yarnpkg.com/socket.io-parser/-/socket.io-parser-4.2.4.tgz#c806966cf7270601e47469ddeec30fbdfda44c83" + integrity sha512-/GbIKmo8ioc+NIWIhwdecY0ge+qVBSMdgxGygevmdHj24bsfgtCmcUUcQ5ZzcylGFHsN3k4HB4Cgkl96KVnuew== + dependencies: + "@socket.io/component-emitter" "~3.1.0" + debug "~4.3.1" + +socket.io@4.6.2: + version "4.6.2" + resolved "https://registry.yarnpkg.com/socket.io/-/socket.io-4.6.2.tgz#d597db077d4df9cbbdfaa7a9ed8ccc3d49439786" + integrity sha512-Vp+lSks5k0dewYTfwgPT9UeGGd+ht7sCpB7p0e83VgO4X/AHYWhXITMrNk/pg8syY2bpx23ptClCQuHhqi2BgQ== dependencies: accepts "~1.3.4" base64id "~2.0.0" debug "~4.3.2" - engine.io "~6.4.1" + engine.io "~6.4.2" socket.io-adapter "~2.5.2" - socket.io-parser "~4.2.1" + socket.io-parser "~4.2.4" socks-proxy-agent@^7.0.0: version "7.0.0" @@ -21102,18 +21110,6 @@ tar@6.1.11: mkdirp "^1.0.3" yallist "^4.0.0" -tar@6.1.15: - version "6.1.15" - resolved "https://registry.yarnpkg.com/tar/-/tar-6.1.15.tgz#c9738b0b98845a3b344d334b8fa3041aaba53a69" - integrity sha512-/zKt9UyngnxIT/EAGYuxaMYgOIJiP81ab9ZfkILq4oNLPFX50qyYmu7jRj9qeXoxmJHjGlbH0+cm2uy1WCs10A== - dependencies: - chownr "^2.0.0" - fs-minipass "^2.0.0" - minipass "^5.0.0" - minizlib "^2.1.1" - mkdirp "^1.0.3" - yallist "^4.0.0" - tar@6.2.1, tar@^6.1.11, tar@^6.1.2: version "6.2.1" resolved "https://registry.yarnpkg.com/tar/-/tar-6.2.1.tgz#717549c541bc3c2af15751bea94b1dd068d4b03a"