From 8c61359b9d940b6f4eaa6d4840d7f53c2e199066 Mon Sep 17 00:00:00 2001 From: Mel O'Hagan Date: Thu, 3 Aug 2023 17:23:15 +0100 Subject: [PATCH 1/3] Allow user specified type casting in MySQL queries --- .vscode/launch.json | 72 ++++++++----------- packages/server/package.json | 2 +- .../server/src/api/controllers/query/index.ts | 4 +- .../tests/{filter.spec.js => filter.spec.ts} | 23 +++--- packages/server/src/integrations/mysql.ts | 11 ++- packages/server/src/threads/definitions.ts | 6 ++ packages/server/src/threads/query.ts | 31 +++++++- 7 files changed, 94 insertions(+), 55 deletions(-) rename packages/server/src/automations/tests/{filter.spec.js => filter.spec.ts} (76%) diff --git a/.vscode/launch.json b/.vscode/launch.json index 8cb49d5825..6c0089bb6b 100644 --- a/.vscode/launch.json +++ b/.vscode/launch.json @@ -1,42 +1,32 @@ { - // Use IntelliSense to learn about possible attributes. - // Hover to view descriptions of existing attributes. - // For more information, visit: https://go.microsoft.com/fwlink/?linkid=830387 - "version": "0.2.0", - "configurations": [ - { - "name": "Budibase Server", - "type": "node", - "request": "launch", - "runtimeArgs": [ - "--nolazy", - "-r", - "ts-node/register/transpile-only" - ], - "args": [ - "${workspaceFolder}/packages/server/src/index.ts" - ], - "cwd": "${workspaceFolder}/packages/server" - }, - { - "name": "Budibase Worker", - "type": "node", - "request": "launch", - "runtimeArgs": [ - "--nolazy", - "-r", - "ts-node/register/transpile-only" - ], - "args": [ - "${workspaceFolder}/packages/worker/src/index.ts" - ], - "cwd": "${workspaceFolder}/packages/worker" - }, - ], - "compounds": [ - { - "name": "Start Budibase", - "configurations": ["Budibase Server", "Budibase Worker"] - } - ] -} \ No newline at end of file + // Use IntelliSense to learn about possible attributes. + // Hover to view descriptions of existing attributes. + // For more information, visit: https://go.microsoft.com/fwlink/?linkid=830387 + "version": "0.2.0", + "configurations": [ + { + "name": "Budibase Server", + "type": "node", + "request": "launch", + "runtimeVersion": "14.20.1", + "runtimeArgs": ["--nolazy", "-r", "ts-node/register/transpile-only"], + "args": ["${workspaceFolder}/packages/server/src/index.ts"], + "cwd": "${workspaceFolder}/packages/server" + }, + { + "name": "Budibase Worker", + "type": "node", + "request": "launch", + "runtimeVersion": "14.20.1", + "runtimeArgs": ["--nolazy", "-r", "ts-node/register/transpile-only"], + "args": ["${workspaceFolder}/packages/worker/src/index.ts"], + "cwd": "${workspaceFolder}/packages/worker" + } + ], + "compounds": [ + { + "name": "Start Budibase", + "configurations": ["Budibase Server", "Budibase Worker"] + } + ] +} diff --git a/packages/server/package.json b/packages/server/package.json index e112c1f572..aedab54a81 100644 --- a/packages/server/package.json +++ b/packages/server/package.json @@ -100,7 +100,7 @@ "memorystream": "0.3.1", "mongodb": "5.7", "mssql": "9.1.1", - "mysql2": "2.3.3", + "mysql2": "3.5.2", "node-fetch": "2.6.7", "object-sizeof": "2.6.1", "open": "8.4.0", diff --git a/packages/server/src/api/controllers/query/index.ts b/packages/server/src/api/controllers/query/index.ts index c32e17bd92..8e13041bf4 100644 --- a/packages/server/src/api/controllers/query/index.ts +++ b/packages/server/src/api/controllers/query/index.ts @@ -120,7 +120,7 @@ export async function preview(ctx: any) { const query = ctx.request.body // preview may not have a queryId as it hasn't been saved, but if it does // this stops dynamic variables from calling the same query - const { fields, parameters, queryVerb, transformer, queryId } = query + const { fields, parameters, queryVerb, transformer, queryId, schema } = query const authConfigCtx: any = getAuthConfig(ctx) @@ -133,6 +133,7 @@ export async function preview(ctx: any) { parameters, transformer, queryId, + schema, // have to pass down to the thread runner - can't put into context now environmentVariables: envVars, ctx: { @@ -228,6 +229,7 @@ async function execute( user: ctx.user, auth: { ...authConfigCtx }, }, + schema: query.schema, } const runFn = () => Runner.run(inputs) diff --git a/packages/server/src/automations/tests/filter.spec.js b/packages/server/src/automations/tests/filter.spec.ts similarity index 76% rename from packages/server/src/automations/tests/filter.spec.js rename to packages/server/src/automations/tests/filter.spec.ts index 5fa5e0e202..c2c5d13e2d 100644 --- a/packages/server/src/automations/tests/filter.spec.js +++ b/packages/server/src/automations/tests/filter.spec.ts @@ -1,11 +1,18 @@ -const setup = require("./utilities") -const { FilterConditions } = require("../steps/filter") +import * as setup from "./utilities" +import { FilterConditions } from "../steps/filter" describe("test the filter logic", () => { - async function checkFilter(field, condition, value, pass = true) { - let res = await setup.runStep(setup.actions.FILTER.stepId, - { field, condition, value } - ) + async function checkFilter( + field: any, + condition: string, + value: any, + pass = true + ) { + let res = await setup.runStep(setup.actions.FILTER.stepId, { + field, + condition, + value, + }) expect(res.result).toEqual(pass) expect(res.success).toEqual(true) } @@ -36,9 +43,9 @@ describe("test the filter logic", () => { it("check date coercion", async () => { await checkFilter( - (new Date()).toISOString(), + new Date().toISOString(), FilterConditions.GREATER_THAN, - (new Date(-10000)).toISOString(), + new Date(-10000).toISOString(), true ) }) diff --git a/packages/server/src/integrations/mysql.ts b/packages/server/src/integrations/mysql.ts index a01f6b71f9..cb71a78013 100644 --- a/packages/server/src/integrations/mysql.ts +++ b/packages/server/src/integrations/mysql.ts @@ -148,7 +148,9 @@ class MySQLIntegration extends Sql implements DatasourcePlus { this.config = { ...config, multipleStatements: true, - typeCast: function (field: any, next: any) { + } + if (!this.config.typeCast) { + this.config.typeCast = function (field: any, next: any) { if ( field.type == "DATETIME" || field.type === "DATE" || @@ -161,7 +163,7 @@ class MySQLIntegration extends Sql implements DatasourcePlus { return field.buffer()?.[0] } return next() - }, + } } } @@ -204,7 +206,10 @@ class MySQLIntegration extends Sql implements DatasourcePlus { async internalQuery( query: SqlQuery, - opts: { connect?: boolean; disableCoercion?: boolean } = { + opts: { + connect?: boolean + disableCoercion?: boolean + } = { connect: true, disableCoercion: false, } diff --git a/packages/server/src/threads/definitions.ts b/packages/server/src/threads/definitions.ts index 21e7ce0b69..dd0891d34a 100644 --- a/packages/server/src/threads/definitions.ts +++ b/packages/server/src/threads/definitions.ts @@ -11,6 +11,12 @@ export interface QueryEvent { queryId: string environmentVariables?: Record ctx?: any + schema?: { + [key: string]: { + name: string + type: string + } + } } export interface QueryVariable { diff --git a/packages/server/src/threads/query.ts b/packages/server/src/threads/query.ts index 9901ddffa6..d3949f7e25 100644 --- a/packages/server/src/threads/query.ts +++ b/packages/server/src/threads/query.ts @@ -8,6 +8,7 @@ import { context, cache, auth } from "@budibase/backend-core" import { getGlobalIDFromUserMetadataID } from "../db/utils" import sdk from "../sdk" import { cloneDeep } from "lodash/fp" +import { SourceName } from "@budibase/types" import { isSQL } from "../integrations/utils" import { interpolateSQL } from "../integrations/queries/sql" @@ -27,6 +28,7 @@ class QueryRunner { hasRerun: boolean hasRefreshedOAuth: boolean hasDynamicVariables: boolean + schema: any constructor(input: QueryEvent, flags = { noRecursiveQuery: false }) { this.datasource = input.datasource @@ -36,6 +38,7 @@ class QueryRunner { this.pagination = input.pagination this.transformer = input.transformer this.queryId = input.queryId + this.schema = input.schema this.noRecursiveQuery = flags.noRecursiveQuery this.cachedVariables = [] // Additional context items for enrichment @@ -50,7 +53,7 @@ class QueryRunner { } async execute(): Promise { - let { datasource, fields, queryVerb, transformer } = this + let { datasource, fields, queryVerb, transformer, schema } = this let datasourceClone = cloneDeep(datasource) let fieldsClone = cloneDeep(fields) @@ -67,6 +70,32 @@ class QueryRunner { datasourceClone.config.authConfigs = updatedConfigs } + if (datasource.source === SourceName.MYSQL && schema) { + datasourceClone.config.typeCast = function (field: any, next: any) { + if (schema[field.name]?.name === field.name) { + if (["LONGLONG", "NEWDECIMAL", "DECIMAL"].includes(field.type)) { + if (schema[field.name]?.type === "number") { + const value = field.string() + return value ? Number(value) : null + } else { + return field.string() + } + } + } + if ( + field.type == "DATETIME" || + field.type === "DATE" || + field.type === "TIMESTAMP" + ) { + return field.string() + } + if (field.type === "BIT" && field.length === 1) { + return field.buffer()?.[0] + } + return next() + } + } + const integration = new Integration(datasourceClone.config) // pre-query, make sure datasource variables are added to parameters From 057761fdd902c8455e15b99316be39e0f973b358 Mon Sep 17 00:00:00 2001 From: Mel O'Hagan Date: Thu, 3 Aug 2023 18:13:51 +0100 Subject: [PATCH 2/3] Update filters types with schemaFields --- .../controls/FilterEditor/FilterDrawer.svelte | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/packages/builder/src/components/design/settings/controls/FilterEditor/FilterDrawer.svelte b/packages/builder/src/components/design/settings/controls/FilterEditor/FilterDrawer.svelte index e8bbb8a354..25a03a93de 100644 --- a/packages/builder/src/components/design/settings/controls/FilterEditor/FilterDrawer.svelte +++ b/packages/builder/src/components/design/settings/controls/FilterEditor/FilterDrawer.svelte @@ -17,7 +17,7 @@ import { generate } from "shortid" import { LuceneUtils, Constants } from "@budibase/frontend-core" import { getFields } from "helpers/searchFields" - import { createEventDispatcher } from "svelte" + import { createEventDispatcher, onMount } from "svelte" export let schemaFields export let filters = [] @@ -64,6 +64,15 @@ }) } + onMount(() => { + parseFilters(filters) + rawFilters.forEach(filter => { + filter.type = + schemaFields.find(field => field.name === filter.field)?.type || + filter.type + }) + }) + // Add field key prefixes and a special metadata filter object to indicate // whether to use the "match all" or "match any" behaviour const enrichFilters = (rawFilters, matchAny) => { From 20f71fadd32fb7f92e8e6ae972ccbf79707e509a Mon Sep 17 00:00:00 2001 From: Mel O'Hagan Date: Thu, 3 Aug 2023 18:52:55 +0100 Subject: [PATCH 3/3] Refactor --- packages/server/src/integrations/mysql.ts | 63 +++++++++++++++++------ packages/server/src/threads/query.ts | 29 ++--------- packages/types/src/sdk/datasources.ts | 6 +++ 3 files changed, 56 insertions(+), 42 deletions(-) diff --git a/packages/server/src/integrations/mysql.ts b/packages/server/src/integrations/mysql.ts index cb71a78013..b88162c59a 100644 --- a/packages/server/src/integrations/mysql.ts +++ b/packages/server/src/integrations/mysql.ts @@ -93,6 +93,21 @@ const SCHEMA: Integration = { }, } +const defaultTypeCasting = function (field: any, next: any) { + if ( + field.type == "DATETIME" || + field.type === "DATE" || + field.type === "TIMESTAMP" || + field.type === "LONGLONG" + ) { + return field.string() + } + if (field.type === "BIT" && field.length === 1) { + return field.buffer()?.[0] + } + return next() +} + export function bindingTypeCoerce(bindings: any[]) { for (let i = 0; i < bindings.length; i++) { const binding = bindings[i] @@ -147,24 +162,9 @@ class MySQLIntegration extends Sql implements DatasourcePlus { delete config.rejectUnauthorized this.config = { ...config, + typeCast: defaultTypeCasting, multipleStatements: true, } - if (!this.config.typeCast) { - this.config.typeCast = function (field: any, next: any) { - if ( - field.type == "DATETIME" || - field.type === "DATE" || - field.type === "TIMESTAMP" || - field.type === "LONGLONG" - ) { - return field.string() - } - if (field.type === "BIT" && field.length === 1) { - return field.buffer()?.[0] - } - return next() - } - } } async testConnection() { @@ -196,6 +196,37 @@ class MySQLIntegration extends Sql implements DatasourcePlus { return `concat(${parts.join(", ")})` } + defineTypeCastingFromSchema(schema: { + [key: string]: { name: string; type: string } + }): void { + if (!schema) { + return + } + this.config.typeCast = function (field: any, next: any) { + if (schema[field.name]?.name === field.name) { + if (["LONGLONG", "NEWDECIMAL", "DECIMAL"].includes(field.type)) { + if (schema[field.name]?.type === "number") { + const value = field.string() + return value ? Number(value) : null + } else { + return field.string() + } + } + } + if ( + field.type == "DATETIME" || + field.type === "DATE" || + field.type === "TIMESTAMP" + ) { + return field.string() + } + if (field.type === "BIT" && field.length === 1) { + return field.buffer()?.[0] + } + return next() + } + } + async connect() { this.client = await mysql.createConnection(this.config) } diff --git a/packages/server/src/threads/query.ts b/packages/server/src/threads/query.ts index d3949f7e25..37720c66ba 100644 --- a/packages/server/src/threads/query.ts +++ b/packages/server/src/threads/query.ts @@ -70,34 +70,11 @@ class QueryRunner { datasourceClone.config.authConfigs = updatedConfigs } - if (datasource.source === SourceName.MYSQL && schema) { - datasourceClone.config.typeCast = function (field: any, next: any) { - if (schema[field.name]?.name === field.name) { - if (["LONGLONG", "NEWDECIMAL", "DECIMAL"].includes(field.type)) { - if (schema[field.name]?.type === "number") { - const value = field.string() - return value ? Number(value) : null - } else { - return field.string() - } - } - } - if ( - field.type == "DATETIME" || - field.type === "DATE" || - field.type === "TIMESTAMP" - ) { - return field.string() - } - if (field.type === "BIT" && field.length === 1) { - return field.buffer()?.[0] - } - return next() - } - } - const integration = new Integration(datasourceClone.config) + // define the type casting from the schema + integration.defineTypeCastingFromSchema?.(schema) + // pre-query, make sure datasource variables are added to parameters const parameters = await this.addDatasourceVariables() diff --git a/packages/types/src/sdk/datasources.ts b/packages/types/src/sdk/datasources.ts index 2391f6e878..d6a0d4a7c8 100644 --- a/packages/types/src/sdk/datasources.ts +++ b/packages/types/src/sdk/datasources.ts @@ -166,6 +166,12 @@ export interface IntegrationBase { delete?(query: any): Promise testConnection?(): Promise getExternalSchema?(): Promise + defineTypeCastingFromSchema?(schema: { + [key: string]: { + name: string + type: string + } + }): void } export interface DatasourcePlus extends IntegrationBase {