diff --git a/packages/backend-core/src/db/constants.ts b/packages/backend-core/src/db/constants.ts deleted file mode 100644 index 69c98fe569..0000000000 --- a/packages/backend-core/src/db/constants.ts +++ /dev/null @@ -1,5 +0,0 @@ -export { - CONSTANT_INTERNAL_ROW_COLS, - CONSTANT_EXTERNAL_ROW_COLS, - isInternalColumnName, -} from "@budibase/shared-core" diff --git a/packages/backend-core/src/db/couch/DatabaseImpl.ts b/packages/backend-core/src/db/couch/DatabaseImpl.ts index 274c09438d..f3c3beeaab 100644 --- a/packages/backend-core/src/db/couch/DatabaseImpl.ts +++ b/packages/backend-core/src/db/couch/DatabaseImpl.ts @@ -13,6 +13,7 @@ import { isDocument, RowResponse, RowValue, + SqlClient, SQLiteDefinition, SqlQueryBinding, } from "@budibase/types" @@ -25,6 +26,7 @@ import { SQLITE_DESIGN_DOC_ID } from "../../constants" import { DDInstrumentedDatabase } from "../instrumentation" import { checkSlashesInUrl } from "../../helpers" import env from "../../environment" +import { sqlLog } from "../../sql/utils" const DATABASE_NOT_FOUND = "Database does not exist." @@ -322,6 +324,7 @@ export class DatabaseImpl implements Database { ): Promise { const dbName = this.name const url = `/${dbName}/${SQLITE_DESIGN_DOC_ID}` + sqlLog(SqlClient.SQL_LITE, sql, parameters) return await this._sqlQuery(url, "POST", { query: sql, args: parameters, diff --git a/packages/backend-core/src/db/couch/index.ts b/packages/backend-core/src/db/couch/index.ts index 932efed3f7..c731d20d6c 100644 --- a/packages/backend-core/src/db/couch/index.ts +++ b/packages/backend-core/src/db/couch/index.ts @@ -2,4 +2,3 @@ export * from "./connections" export * from "./DatabaseImpl" export * from "./utils" export { init, getPouch, getPouchDB, closePouchDB } from "./pouchDB" -export * from "../constants" diff --git a/packages/backend-core/src/sql/sql.ts b/packages/backend-core/src/sql/sql.ts index 615753efc3..e8881c22d4 100644 --- a/packages/backend-core/src/sql/sql.ts +++ b/packages/backend-core/src/sql/sql.ts @@ -3,8 +3,10 @@ import * as dbCore from "../db" import { getNativeSql, isExternalTable, - isIsoDateString, + isValidISODateString, isValidFilter, + sqlLog, + isInvalidISODateString, } from "./utils" import { SqlStatements } from "./sqlStatements" import SqlTableQueryBuilder from "./sqlTable" @@ -38,10 +40,6 @@ const envLimit = environment.SQL_MAX_ROWS : null const BASE_LIMIT = envLimit || 5000 -// these are invalid dates sent by the client, need to convert them to a real max date -const MIN_ISO_DATE = "0000-00-00T00:00:00.000Z" -const MAX_ISO_DATE = "9999-00-00T00:00:00.000Z" - function likeKey(client: string, key: string): string { let start: string, end: string switch (client) { @@ -75,10 +73,10 @@ function parse(input: any) { if (typeof input !== "string") { return input } - if (input === MAX_ISO_DATE || input === MIN_ISO_DATE) { + if (isInvalidISODateString(input)) { return null } - if (isIsoDateString(input)) { + if (isValidISODateString(input)) { return new Date(input.trim()) } return input @@ -938,15 +936,7 @@ class SqlQueryBuilder extends SqlTableQueryBuilder { } log(query: string, values?: SqlQueryBinding) { - if (!environment.SQL_LOGGING_ENABLE) { - return - } - const sqlClient = this.getSqlClient() - let string = `[SQL] [${sqlClient.toUpperCase()}] query="${query}"` - if (values) { - string += ` values="${values.join(", ")}"` - } - console.log(string) + sqlLog(this.getSqlClient(), query, values) } } diff --git a/packages/backend-core/src/sql/utils.ts b/packages/backend-core/src/sql/utils.ts index 45ab510948..e73c6ac445 100644 --- a/packages/backend-core/src/sql/utils.ts +++ b/packages/backend-core/src/sql/utils.ts @@ -2,10 +2,12 @@ import { DocumentType, SqlQuery, Table, TableSourceType } from "@budibase/types" import { DEFAULT_BB_DATASOURCE_ID } from "../constants" import { Knex } from "knex" import { SEPARATOR } from "../db" +import environment from "../environment" const DOUBLE_SEPARATOR = `${SEPARATOR}${SEPARATOR}` const ROW_ID_REGEX = /^\[.*]$/g const ENCODED_SPACE = encodeURIComponent(" ") +const ISO_DATE_REGEX = /^\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}.\d{3}Z$/ export function isExternalTableID(tableId: string) { return tableId.startsWith(DocumentType.DATASOURCE + SEPARATOR) @@ -120,15 +122,38 @@ export function breakRowIdField(_id: string | { _id: string }): any[] { } } -export function isIsoDateString(str: string) { +export function isInvalidISODateString(str: string) { const trimmedValue = str.trim() - if (!/^\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}.\d{3}Z$/.test(trimmedValue)) { + if (!ISO_DATE_REGEX.test(trimmedValue)) { return false } let d = new Date(trimmedValue) + return isNaN(d.getTime()) +} + +export function isValidISODateString(str: string) { + const trimmedValue = str.trim() + if (!ISO_DATE_REGEX.test(trimmedValue)) { + return false + } + let d = new Date(trimmedValue) + if (isNaN(d.getTime())) { + return false + } return d.toISOString() === trimmedValue } export function isValidFilter(value: any) { return value != null && value !== "" } + +export function sqlLog(client: string, query: string, values?: any[]) { + if (!environment.SQL_LOGGING_ENABLE) { + return + } + let string = `[SQL] [${client.toUpperCase()}] query="${query}"` + if (values) { + string += ` values="${values.join(", ")}"` + } + console.log(string) +} diff --git a/packages/backend-core/tests/core/utilities/jestUtils.ts b/packages/backend-core/tests/core/utilities/jestUtils.ts index 4a3da8db8c..a49c2a795e 100644 --- a/packages/backend-core/tests/core/utilities/jestUtils.ts +++ b/packages/backend-core/tests/core/utilities/jestUtils.ts @@ -1,4 +1,7 @@ -import { db } from "../../../src" +import { + CONSTANT_EXTERNAL_ROW_COLS, + CONSTANT_INTERNAL_ROW_COLS, +} from "@budibase/shared-core" export function expectFunctionWasCalledTimesWith( jestFunction: any, @@ -11,7 +14,7 @@ export function expectFunctionWasCalledTimesWith( } export const expectAnyInternalColsAttributes: { - [K in (typeof db.CONSTANT_INTERNAL_ROW_COLS)[number]]: any + [K in (typeof CONSTANT_INTERNAL_ROW_COLS)[number]]: any } = { tableId: expect.anything(), type: expect.anything(), @@ -22,7 +25,7 @@ export const expectAnyInternalColsAttributes: { } export const expectAnyExternalColsAttributes: { - [K in (typeof db.CONSTANT_EXTERNAL_ROW_COLS)[number]]: any + [K in (typeof CONSTANT_EXTERNAL_ROW_COLS)[number]]: any } = { tableId: expect.anything(), _id: expect.anything(), diff --git a/packages/server/src/api/routes/tests/search.spec.ts b/packages/server/src/api/routes/tests/search.spec.ts index 145db9b4a3..bdc8fc1af6 100644 --- a/packages/server/src/api/routes/tests/search.spec.ts +++ b/packages/server/src/api/routes/tests/search.spec.ts @@ -2167,6 +2167,35 @@ describe.each([ } ) + describe.each([ + { low: "2024-07-03T00:00:00.000Z", high: "9999-00-00T00:00:00.000Z" }, + { low: "2024-07-03T00:00:00.000Z", high: "9998-00-00T00:00:00.000Z" }, + { low: "0000-00-00T00:00:00.000Z", high: "2024-07-04T00:00:00.000Z" }, + { low: "0001-00-00T00:00:00.000Z", high: "2024-07-04T00:00:00.000Z" }, + ])("date special cases", ({ low, high }) => { + const earlyDate = "2024-07-03T10:00:00.000Z", + laterDate = "2024-07-03T11:00:00.000Z" + beforeAll(async () => { + table = await createTable({ + date: { + name: "date", + type: FieldType.DATETIME, + }, + }) + await createRows([{ date: earlyDate }, { date: laterDate }]) + }) + + it("should be able to handle a date search", async () => { + await expectSearch({ + query: { + range: { + "1:date": { low, high }, + }, + }, + }).toContainExactly([{ date: earlyDate }, { date: laterDate }]) + }) + }) + describe.each([ "名前", // Japanese for "name" "Benutzer-ID", // German for "user ID", includes a hyphen diff --git a/packages/server/src/sdk/app/rows/search/external.ts b/packages/server/src/sdk/app/rows/search/external.ts index 93c46d8cc3..1b6638f671 100644 --- a/packages/server/src/sdk/app/rows/search/external.ts +++ b/packages/server/src/sdk/app/rows/search/external.ts @@ -16,9 +16,9 @@ import { breakExternalTableId, breakRowIdField, } from "../../../../integrations/utils" -import { utils } from "@budibase/shared-core" +import { utils, CONSTANT_EXTERNAL_ROW_COLS } from "@budibase/shared-core" import { ExportRowsParams, ExportRowsResult } from "./types" -import { db, HTTPError } from "@budibase/backend-core" +import { HTTPError } from "@budibase/backend-core" import pick from "lodash/pick" import { outputProcessing } from "../../../../utilities/rowProcessor" import sdk from "../../../" @@ -99,7 +99,7 @@ export async function search( } if (options.fields) { - const fields = [...options.fields, ...db.CONSTANT_EXTERNAL_ROW_COLS] + const fields = [...options.fields, ...CONSTANT_EXTERNAL_ROW_COLS] rows = rows.map((r: any) => pick(r, fields)) } diff --git a/packages/server/src/sdk/app/rows/search/internal.ts b/packages/server/src/sdk/app/rows/search/internal.ts index 906ca016d1..097b16b104 100644 --- a/packages/server/src/sdk/app/rows/search/internal.ts +++ b/packages/server/src/sdk/app/rows/search/internal.ts @@ -1,4 +1,5 @@ -import { context, db, HTTPError } from "@budibase/backend-core" +import { context, HTTPError } from "@budibase/backend-core" +import { CONSTANT_INTERNAL_ROW_COLS } from "@budibase/shared-core" import env from "../../../../environment" import { fullSearch, paginatedSearch } from "./utils" import { getRowParams, InternalTables } from "../../../../db/utils" @@ -74,7 +75,7 @@ export async function search( } if (options.fields) { - const fields = [...options.fields, ...db.CONSTANT_INTERNAL_ROW_COLS] + const fields = [...options.fields, ...CONSTANT_INTERNAL_ROW_COLS] response.rows = response.rows.map((r: any) => pick(r, fields)) } diff --git a/packages/server/src/sdk/app/tables/migration.ts b/packages/server/src/sdk/app/tables/migration.ts index d0decf01f6..8e084381aa 100644 --- a/packages/server/src/sdk/app/tables/migration.ts +++ b/packages/server/src/sdk/app/tables/migration.ts @@ -18,6 +18,7 @@ import sdk from "../../../sdk" import { isExternalTableID } from "../../../integrations/utils" import { EventType, updateLinks } from "../../../db/linkedRows" import { cloneDeep } from "lodash" +import { isInternalColumnName } from "@budibase/shared-core" export interface MigrationResult { tablesUpdated: Table[] @@ -36,7 +37,7 @@ export async function migrate( throw new BadRequestError(`Column name cannot be empty`) } - if (dbCore.isInternalColumnName(newColumnName)) { + if (isInternalColumnName(newColumnName)) { throw new BadRequestError(`Column name cannot be a reserved column name`) } diff --git a/packages/server/src/sdk/app/views/index.ts b/packages/server/src/sdk/app/views/index.ts index fce57a390d..3bdfec7448 100644 --- a/packages/server/src/sdk/app/views/index.ts +++ b/packages/server/src/sdk/app/views/index.ts @@ -6,9 +6,13 @@ import { ViewV2, ViewV2Enriched, } from "@budibase/types" -import { HTTPError, db as dbCore } from "@budibase/backend-core" +import { HTTPError } from "@budibase/backend-core" import { features } from "@budibase/pro" -import { helpers } from "@budibase/shared-core" +import { + helpers, + CONSTANT_EXTERNAL_ROW_COLS, + CONSTANT_INTERNAL_ROW_COLS, +} from "@budibase/shared-core" import { cloneDeep } from "lodash/fp" import * as utils from "../../../db/utils" @@ -144,8 +148,8 @@ export function allowedFields(view: View | ViewV2) { const fieldSchema = view.schema![key] return fieldSchema.visible && !fieldSchema.readonly }), - ...dbCore.CONSTANT_EXTERNAL_ROW_COLS, - ...dbCore.CONSTANT_INTERNAL_ROW_COLS, + ...CONSTANT_EXTERNAL_ROW_COLS, + ...CONSTANT_INTERNAL_ROW_COLS, ] }