MirrorSave
/
budibase
mirror of https://github.com/Budibase/budibase.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Changing how SQL vars are generated so that when new SQL implementations are added they must implement a generation mechanism.

This commit is contained in:
Michael Drury 2022-03-02 22:45:10 +00:00
parent 1b45a9190d
commit 546bbc2ff6
8 changed files with 149 additions and 1088 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
packages/server/src/integrations/base/datasourcePlus.ts
View File

@ -5,5 +5,8 @@ export interface DatasourcePlus extends IntegrationBase {
tables: Record<string, Table>
schemaErrors: Record<string, string>
// if the datasource supports the use of bindings directly (to protect against SQL injection)
// this returns the format of the identifier
getBindingIdentifier(): string
buildSchema(datasourceId: string, entities: Record<string, Table>): any
}

7
packages/server/src/integrations/googlesheets.ts
View File

@ -6,11 +6,10 @@ import {
} from "../definitions/datasource"
import { OAuth2Client } from "google-auth-library"
import { DatasourcePlus } from "./base/datasourcePlus"
import { Row, Table, TableSchema } from "../definitions/common"
import { Table, TableSchema } from "../definitions/common"
import { buildExternalTableId } from "./utils"
import { DataSourceOperation, FieldTypes } from "../constants"
import { GoogleSpreadsheet } from "google-spreadsheet"
import { table } from "console"
module GoogleSheetsModule {
const { getGlobalDB } = require("@budibase/backend-core/tenancy")
@ -112,6 +111,10 @@ module GoogleSheetsModule {
this.client = new GoogleSpreadsheet(spreadsheetId)
}
getBindingIdentifier() {
return ""
}
/**
* Pull the spreadsheet ID out from a valid google sheets URL
* @param spreadsheetId - the URL or standard spreadsheetId of the google sheet

71
packages/server/src/integrations/microsoftSqlServer.ts
View File

@ -79,34 +79,9 @@ module MSSQLModule {
},
}
async function internalQuery(
client: any,
query: SqlQuery,
operation: string | undefined = undefined
) {
const request = client.request()
try {
if (Array.isArray(query.bindings)) {
let count = 0
for (let binding of query.bindings) {
request.input(`p${count++}`, binding)
}
}
// this is a hack to get the inserted ID back,
// no way to do this with Knex nicely
const sql =
operation === Operation.CREATE
? `${query.sql}; SELECT SCOPE_IDENTITY() AS id;`
: query.sql
return await request.query(sql)
} catch (err) {
// @ts-ignore
throw new Error(err)
}
}
class SqlServerIntegration extends Sql implements DatasourcePlus {
private readonly config: MSSQLConfig
private index: number = 0
static pool: any
public tables: Record<string, Table> = {}
public schemaErrors: Record<string, string> = {}
@ -121,6 +96,33 @@ module MSSQLModule {
TABLES_SQL =
"SELECT * FROM INFORMATION_SCHEMA.TABLES WHERE TABLE_TYPE='BASE TABLE'"
async internalQuery(
query: SqlQuery,
operation: string | undefined = undefined
) {
const client = this.client
const request = client.request()
this.index = 0
try {
if (Array.isArray(query.bindings)) {
let count = 0
for (let binding of query.bindings) {
request.input(`p${count++}`, binding)
}
}
// this is a hack to get the inserted ID back,
// no way to do this with Knex nicely
const sql =
operation === Operation.CREATE
? `${query.sql}; SELECT SCOPE_IDENTITY() AS id;`
: query.sql
return await request.query(sql)
} catch (err) {
// @ts-ignore
throw new Error(err)
}
}
getDefinitionSQL(tableName: string) {
return `select *
from INFORMATION_SCHEMA.COLUMNS
@ -165,6 +167,10 @@ module MSSQLModule {
}
}
getBindingIdentifier(): string {
return `(@p${this.index++})`
}
async connect() {
try {
this.client = await this.pool.connect()
@ -175,7 +181,7 @@ module MSSQLModule {
}
async runSQL(sql: string) {
return (await internalQuery(this.client, getSqlQuery(sql))).recordset
return (await this.internalQuery(getSqlQuery(sql))).recordset
}
/**
@ -238,33 +244,32 @@ module MSSQLModule {
async read(query: SqlQuery | string) {
await this.connect()
const response = await internalQuery(this.client, getSqlQuery(query))
const response = await this.internalQuery(getSqlQuery(query))
return response.recordset
}
async create(query: SqlQuery | string) {
await this.connect()
const response = await internalQuery(this.client, getSqlQuery(query))
const response = await this.internalQuery(getSqlQuery(query))
return response.recordset || [{ created: true }]
}
async update(query: SqlQuery | string) {
await this.connect()
const response = await internalQuery(this.client, getSqlQuery(query))
const response = await this.internalQuery(getSqlQuery(query))
return response.recordset || [{ updated: true }]
}
async delete(query: SqlQuery | string) {
await this.connect()
const response = await internalQuery(this.client, getSqlQuery(query))
const response = await this.internalQuery(getSqlQuery(query))
return response.recordset || [{ deleted: true }]
}
async query(json: QueryJson) {
await this.connect()
const operation = this._operation(json)
const queryFn = (query: any, op: string) =>
internalQuery(this.client, query, op)
const queryFn = (query: any, op: string) => this.internalQuery(query, op)
const processFn = (result: any) =>
result.recordset ? result.recordset : [{ [operation]: true }]
return this.queryWithReturning(json, queryFn, processFn)

74
packages/server/src/integrations/mysql.ts
View File

@ -80,33 +80,6 @@ module MySQLModule {
},
}
function internalQuery(
client: any,
query: SqlQuery,
connect: boolean = true
): Promise<any[] | any> {
// Node MySQL is callback based, so we must wrap our call in a promise
return new Promise((resolve, reject) => {
if (connect) {
client.connect()
}
return client.query(
query.sql,
query.bindings || {},
(error: any, results: object[]) => {
if (error) {
reject(error)
} else {
resolve(results)
}
if (connect) {
client.end()
}
}
)
})
}
class MySQLIntegration extends Sql implements DatasourcePlus {
private config: MySQLConfig
private readonly client: any
@ -122,14 +95,44 @@ module MySQLModule {
this.client = mysql.createConnection(config)
}
getBindingIdentifier(): string {
return "?"
}
internalQuery(
query: SqlQuery,
connect: boolean = true
): Promise<any[] | any> {
const client = this.client
// Node MySQL is callback based, so we must wrap our call in a promise
return new Promise((resolve, reject) => {
if (connect) {
client.connect()
}
return client.query(
query.sql,
query.bindings || {},
(error: any, results: object[]) => {
if (error) {
reject(error)
} else {
resolve(results)
}
if (connect) {
client.end()
}
}
)
})
}
async buildSchema(datasourceId: string, entities: Record<string, Table>) {
const tables: { [key: string]: Table } = {}
const database = this.config.database
this.client.connect()
// get the tables first
const tablesResp = await internalQuery(
this.client,
const tablesResp = await this.internalQuery(
{ sql: "SHOW TABLES;" },
false
)
@ -141,8 +144,7 @@ module MySQLModule {
for (let tableName of tableNames) {
const primaryKeys = []
const schema: TableSchema = {}
const descResp = await internalQuery(
this.client,
const descResp = await this.internalQuery(
{ sql: `DESCRIBE \`${tableName}\`;` },
false
)
@ -182,27 +184,27 @@ module MySQLModule {
}
async create(query: SqlQuery | string) {
const results = await internalQuery(this.client, getSqlQuery(query))
const results = await this.internalQuery(getSqlQuery(query))
return results.length ? results : [{ created: true }]
}
async read(query: SqlQuery | string) {
return internalQuery(this.client, getSqlQuery(query))
return this.internalQuery(getSqlQuery(query))
}
async update(query: SqlQuery | string) {
const results = await internalQuery(this.client, getSqlQuery(query))
const results = await this.internalQuery(getSqlQuery(query))
return results.length ? results : [{ updated: true }]
}
async delete(query: SqlQuery | string) {
const results = await internalQuery(this.client, getSqlQuery(query))
const results = await this.internalQuery(getSqlQuery(query))
return results.length ? results : [{ deleted: true }]
}
async query(json: QueryJson) {
this.client.connect()
const queryFn = (query: any) => internalQuery(this.client, query, false)
const queryFn = (query: any) => this.internalQuery(query, false)
const output = await this.queryWithReturning(json, queryFn)
this.client.end()
return output

6
packages/server/src/integrations/oracle.ts
View File

@ -137,6 +137,7 @@ module OracleModule {
class OracleIntegration extends Sql implements DatasourcePlus {
private readonly config: OracleConfig
private index: number = 1
public tables: Record<string, Table> = {}
public schemaErrors: Record<string, string> = {}
@ -174,6 +175,10 @@ module OracleModule {
this.config = config
}
getBindingIdentifier(): string {
return `:${this.index++}`
}
/**
* Map the flat tabular columns and constraints data into a nested object
*/
@ -343,6 +348,7 @@ module OracleModule {
private async internalQuery<T>(query: SqlQuery): Promise<Result<T>> {
let connection
try {
this.index = 1
connection = await this.getConnection()
const options: ExecuteOptions = { autoCommit: true }

59
packages/server/src/integrations/postgres.ts
View File

@ -103,30 +103,11 @@ module PostgresModule {
},
}
async function internalQuery(client: any, query: SqlQuery) {
// need to handle a specific issue with json data types in postgres,
// new lines inside the JSON data will break it
if (query && query.sql) {
const matches = query.sql.match(JSON_REGEX)
if (matches && matches.length > 0) {
for (let match of matches) {
const escaped = escapeDangerousCharacters(match)
query.sql = query.sql.replace(match, escaped)
}
}
}
try {
return await client.query(query.sql, query.bindings || [])
} catch (err) {
// @ts-ignore
throw new Error(err)
}
}
class PostgresIntegration extends Sql implements DatasourcePlus {
static pool: any
private readonly client: any
private readonly config: PostgresConfig
private index: number = 1
public tables: Record<string, Table> = {}
public schemaErrors: Record<string, string> = {}
@ -163,6 +144,32 @@ module PostgresModule {
this.setSchema()
}
getBindingIdentifier(): string {
return `$${this.index++}`
}
async internalQuery(query: SqlQuery) {
const client = this.client
this.index = 1
// need to handle a specific issue with json data types in postgres,
// new lines inside the JSON data will break it
if (query && query.sql) {
const matches = query.sql.match(JSON_REGEX)
if (matches && matches.length > 0) {
for (let match of matches) {
const escaped = escapeDangerousCharacters(match)
query.sql = query.sql.replace(match, escaped)
}
}
}
try {
return await client.query(query.sql, query.bindings || [])
} catch (err) {
// @ts-ignore
throw new Error(err)
}
}
setSchema() {
if (!this.config.schema) {
this.config.schema = "public"
@ -241,22 +248,22 @@ module PostgresModule {
}
async create(query: SqlQuery | string) {
const response = await internalQuery(this.client, getSqlQuery(query))
const response = await this.internalQuery(getSqlQuery(query))
return response.rows.length ? response.rows : [{ created: true }]
}
async read(query: SqlQuery | string) {
const response = await internalQuery(this.client, getSqlQuery(query))
const response = await this.internalQuery(getSqlQuery(query))
return response.rows
}
async update(query: SqlQuery | string) {
const response = await internalQuery(this.client, getSqlQuery(query))
const response = await this.internalQuery(getSqlQuery(query))
return response.rows.length ? response.rows : [{ updated: true }]
}
async delete(query: SqlQuery | string) {
const response = await internalQuery(this.client, getSqlQuery(query))
const response = await this.internalQuery(getSqlQuery(query))
return response.rows.length ? response.rows : [{ deleted: true }]
}
@ -266,11 +273,11 @@ module PostgresModule {
if (Array.isArray(input)) {
const responses = []
for (let query of input) {
responses.push(await internalQuery(this.client, query))
responses.push(await this.internalQuery(query))
}
return responses
} else {
const response = await internalQuery(this.client, input)
const response = await this.internalQuery(input)
return response.rows.length ? response.rows : [{ [operation]: true }]
}
}

36
packages/server/src/threads/query.js
View File

@ -2,7 +2,6 @@ const threadUtils = require("./utils")
threadUtils.threadSetup()
const ScriptRunner = require("../utilities/scriptRunner")
const { integrations } = require("../integrations")
const { SourceNames } = require("../definitions/datasource")
const {
processStringSync,
findHBSBlocks,
@ -28,29 +27,12 @@ class QueryRunner {
this.hasRerun = false
}
interpolateSQL(fields, parameters) {
let { datasource } = this
interpolateSQL(fields, parameters, integration) {
let sql = fields.sql
const bindings = findHBSBlocks(sql)
let index = 1
let variables = []
for (let binding of bindings) {
let variable
switch (datasource.source) {
case SourceNames.POSTGRES:
variable = `$${index}`
break
case SourceNames.SQL_SERVER:
variable = `(@p${index - 1})`
break
case SourceNames.MYSQL:
variable = "?"
break
case SourceNames.ORACLE:
variable = `:${index}`
break
}
index++
let variable = integration.getBindingIdentifier()
variables.push(binding)
sql = sql.replace(binding, variable)
}
@ -62,12 +44,18 @@ class QueryRunner {
async execute() {
let { datasource, fields, queryVerb, transformer } = this
const Integration = integrations[datasource.source]
if (!Integration) {
throw "Integration type does not exist."
}
const integration = new Integration(datasource.config)
// pre-query, make sure datasource variables are added to parameters
const parameters = await this.addDatasourceVariables()
let query
// handle SQL injections by interpolating the variables
if (isSQL(datasource)) {
query = this.interpolateSQL(fields, parameters)
query = this.interpolateSQL(fields, parameters, integration)
} else {
query = this.enrichQueryFields(fields, parameters)
}
@ -77,12 +65,6 @@ class QueryRunner {
query.paginationValues = this.pagination
}
const Integration = integrations[datasource.source]
if (!Integration) {
throw "Integration type does not exist."
}
const integration = new Integration(datasource.config)
let output = threadUtils.formatResponse(await integration[queryVerb](query))
let rows = output,
info = undefined,

981
packages/server/yarn.lock
View File

File diff suppressed because it is too large Load Diff