import validateJs from "validate.js" import dayjs from "dayjs" import cloneDeep from "lodash/fp/cloneDeep" import { Datasource, DatasourcePlusQueryResponse, FieldConstraints, FieldType, QueryJson, Row, SourceName, Table, TableSchema, SqlClient, ArrayOperator, ViewV2, } from "@budibase/types" import { makeExternalQuery } from "../../../integrations/base/query" import { Format } from "../../../api/controllers/view/exporters" import sdk from "../.." import { extractViewInfoFromID, isRelationshipColumn } from "../../../db/utils" import { isSQL } from "../../../integrations/utils" import { docIds, sql } from "@budibase/backend-core" import { getTableFromSource } from "../../../api/controllers/row/utils" import env from "../../../environment" const SQL_CLIENT_SOURCE_MAP: Record = { [SourceName.POSTGRES]: SqlClient.POSTGRES, [SourceName.MYSQL]: SqlClient.MY_SQL, [SourceName.SQL_SERVER]: SqlClient.MS_SQL, [SourceName.ORACLE]: SqlClient.ORACLE, [SourceName.DYNAMODB]: undefined, [SourceName.MONGODB]: undefined, [SourceName.ELASTICSEARCH]: undefined, [SourceName.COUCHDB]: undefined, [SourceName.S3]: undefined, [SourceName.AIRTABLE]: undefined, [SourceName.ARANGODB]: undefined, [SourceName.REST]: undefined, [SourceName.FIRESTORE]: undefined, [SourceName.GOOGLE_SHEETS]: undefined, [SourceName.REDIS]: undefined, [SourceName.SNOWFLAKE]: undefined, [SourceName.BUDIBASE]: undefined, } const XSS_INPUT_REGEX = /[<>;"'(){}]|--|\/\*|\*\/|union|select|insert|drop|delete|update|exec|script/i export function getSQLClient(datasource: Datasource): SqlClient { if (!isSQL(datasource)) { throw new Error("Cannot get SQL Client for non-SQL datasource") } const lookup = SQL_CLIENT_SOURCE_MAP[datasource.source] if (lookup) { return lookup } throw new Error("Unable to determine client for SQL datasource") } export function processRowCountResponse( response: DatasourcePlusQueryResponse ): number { if ( response && response.length === 1 && sql.COUNT_FIELD_NAME in response[0] ) { const total = response[0][sql.COUNT_FIELD_NAME] return typeof total === "number" ? total : parseInt(total) } else { throw new Error("Unable to count rows in query - no count response") } } export async function getDatasourceAndQuery( json: QueryJson ): Promise { const datasourceId = json.endpoint.datasourceId const datasource = await sdk.datasources.get(datasourceId) const table = datasource.entities?.[json.endpoint.entityId] if (!json.meta && table) { json.meta = { table, } } return makeExternalQuery(datasource, json) } export function cleanExportRows( rows: Row[], schema: TableSchema, format: string, columns?: string[], customHeaders: { [key: string]: string } = {} ) { let cleanRows = [...rows] const relationships = Object.entries(schema) .filter((entry: any[]) => entry[1].type === FieldType.LINK) .map(entry => entry[0]) relationships.forEach(column => { cleanRows.forEach(row => { delete row[column] }) delete schema[column] }) if (format === Format.CSV) { // Intended to append empty values in export const schemaKeys = Object.keys(schema) for (let key of schemaKeys) { if (columns?.length && columns.indexOf(key) > 0) { continue } for (let row of cleanRows) { if (row[key] == null) { row[key] = undefined } } } } else if (format === Format.JSON) { // Replace row keys with custom headers for (let row of cleanRows) { renameKeys(customHeaders, row) } } return cleanRows } function renameKeys(keysMap: { [key: string]: any }, row: any) { for (const key in keysMap) { Object.defineProperty( row, keysMap[key], Object.getOwnPropertyDescriptor(row, key) || {} ) delete row[key] } } function isForeignKey(key: string, table: Table) { const relationships = Object.values(table.schema).filter(isRelationshipColumn) return relationships.some( relationship => (relationship as any).foreignKey === key ) } export async function validate({ source, row, }: { source: Table | ViewV2 row: Row }): Promise<{ valid: boolean errors: Record }> { const table = await getTableFromSource(source) const errors: Record = {} const disallowArrayTypes = [ FieldType.ATTACHMENT_SINGLE, FieldType.BB_REFERENCE_SINGLE, ] for (let fieldName of Object.keys(table.schema)) { const column = table.schema[fieldName] const constraints = cloneDeep(column.constraints) const type = column.type // foreign keys are likely to be enriched if (isForeignKey(fieldName, table)) { continue } // formulas shouldn't validated, data will be deleted anyway if (type === FieldType.FORMULA || column.autocolumn) { continue } // special case for options, need to always allow unselected (empty) if (type === FieldType.OPTIONS && constraints?.inclusion) { constraints.inclusion.push(null as any, "") } if (disallowArrayTypes.includes(type) && Array.isArray(row[fieldName])) { errors[fieldName] = `Cannot accept arrays` } let res // Validate.js doesn't seem to handle array if (type === FieldType.ARRAY && row[fieldName]) { if (row[fieldName].length) { if (!Array.isArray(row[fieldName])) { row[fieldName] = row[fieldName].split(",") } row[fieldName].map((val: any) => { if ( !constraints?.inclusion?.includes(val) && constraints?.inclusion?.length !== 0 ) { errors[fieldName] = "Field not in list" } }) } else if (constraints?.presence && row[fieldName].length === 0) { // non required MultiSelect creates an empty array, which should not throw errors errors[fieldName] = [`${fieldName} is required`] } } else if ( (type === FieldType.ATTACHMENTS || type === FieldType.JSON) && typeof row[fieldName] === "string" ) { // this should only happen if there is an error try { const json = JSON.parse(row[fieldName]) if (type === FieldType.ATTACHMENTS) { if (Array.isArray(json)) { row[fieldName] = json } else { errors[fieldName] = [`Must be an array`] } } } catch (err) { errors[fieldName] = [`Contains invalid JSON`] } } else if (type === FieldType.DATETIME && column.timeOnly) { res = validateTimeOnlyField(fieldName, row[fieldName], constraints) } else { res = validateJs.single(row[fieldName], constraints) } if (env.XSS_SAFE_MODE && typeof row[fieldName] === "string") { if (XSS_INPUT_REGEX.test(row[fieldName])) { errors[fieldName] = [ "Input not sanitised - potentially vulnerable to XSS", ] } } if (res) errors[fieldName] = res } return { valid: Object.keys(errors).length === 0, errors } } function validateTimeOnlyField( fieldName: string, value: any, constraints: FieldConstraints | undefined ) { let res if (value && !value.match(/^(\d+)(:[0-5]\d){1,2}$/)) { res = [`"${fieldName}" is not a valid time`] } else if (constraints) { let castedValue = value const stringTimeToDate = (value: string) => { const [hour, minute, second] = value.split(":").map((x: string) => +x) let date = dayjs("2000-01-01T00:00:00.000Z").hour(hour).minute(minute) if (!isNaN(second)) { date = date.second(second) } return date } if (castedValue) { castedValue = stringTimeToDate(castedValue) } let castedConstraints = cloneDeep(constraints) let earliest, latest let easliestTimeString: string, latestTimeString: string if (castedConstraints.datetime?.earliest) { easliestTimeString = castedConstraints.datetime.earliest if (dayjs(castedConstraints.datetime.earliest).isValid()) { easliestTimeString = dayjs(castedConstraints.datetime.earliest).format( "HH:mm" ) } earliest = stringTimeToDate(easliestTimeString) } if (castedConstraints.datetime?.latest) { latestTimeString = castedConstraints.datetime.latest if (dayjs(castedConstraints.datetime.latest).isValid()) { latestTimeString = dayjs(castedConstraints.datetime.latest).format( "HH:mm" ) } latest = stringTimeToDate(latestTimeString) } if (earliest && latest && earliest.isAfter(latest)) { latest = latest.add(1, "day") if (earliest.isAfter(castedValue)) { castedValue = castedValue.add(1, "day") } } if (earliest || latest) { castedConstraints.datetime = { earliest: earliest?.toISOString() || "", latest: latest?.toISOString() || "", } } let jsValidation = validateJs.single( castedValue?.toISOString(), castedConstraints ) jsValidation = jsValidation?.map((m: string) => m ?.replace( castedConstraints.datetime?.earliest || "", easliestTimeString || "" ) .replace( castedConstraints.datetime?.latest || "", latestTimeString || "" ) ) if (jsValidation) { res ??= [] res.push(...jsValidation) } } return res } // type-guard check export function isArrayFilter(operator: any): operator is ArrayOperator { return Object.values(ArrayOperator).includes(operator) } export function tryExtractingTableAndViewId(tableOrViewId: string) { if (docIds.isViewId(tableOrViewId)) { return { tableId: extractViewInfoFromID(tableOrViewId).tableId, viewId: tableOrViewId, } } return { tableId: tableOrViewId } } export function getSource(tableOrViewId: string) { if (docIds.isViewId(tableOrViewId)) { return sdk.views.get(tableOrViewId) } return sdk.tables.getTable(tableOrViewId) }