From ac27ba4c81a0ae39f2307d3d8c95fd0ce457d744 Mon Sep 17 00:00:00 2001 From: mike12345567 Date: Wed, 27 Oct 2021 17:32:02 +0100 Subject: [PATCH 01/12] Adding docker compose for ms-sql with products, tasks table setup. --- .../integrations/mssql/data/Dockerfile | 9 +++++ .../integrations/mssql/data/entrypoint.sh | 24 +++++++++++++ .../scripts/integrations/mssql/data/setup.sql | 34 +++++++++++++++++++ .../integrations/mssql/docker-compose.yaml | 12 +++++++ .../scripts/integrations/mssql/reset.sh | 3 ++ 5 files changed, 82 insertions(+) create mode 100644 packages/server/scripts/integrations/mssql/data/Dockerfile create mode 100644 packages/server/scripts/integrations/mssql/data/entrypoint.sh create mode 100644 packages/server/scripts/integrations/mssql/data/setup.sql create mode 100644 packages/server/scripts/integrations/mssql/docker-compose.yaml create mode 100755 packages/server/scripts/integrations/mssql/reset.sh diff --git a/packages/server/scripts/integrations/mssql/data/Dockerfile b/packages/server/scripts/integrations/mssql/data/Dockerfile new file mode 100644 index 0000000000..8ac56409a0 --- /dev/null +++ b/packages/server/scripts/integrations/mssql/data/Dockerfile @@ -0,0 +1,9 @@ +FROM mcr.microsoft.com/mssql/server + +ENV ACCEPT_EULA=Y +ENV SA_PASSWORD=Passw0rd + +COPY ./data / + +ENTRYPOINT [ "/bin/bash", "entrypoint.sh" ] +CMD [ "/opt/mssql/bin/sqlservr" ] diff --git a/packages/server/scripts/integrations/mssql/data/entrypoint.sh b/packages/server/scripts/integrations/mssql/data/entrypoint.sh new file mode 100644 index 0000000000..04780d085e --- /dev/null +++ b/packages/server/scripts/integrations/mssql/data/entrypoint.sh @@ -0,0 +1,24 @@ +#!/bin/bash +set -e + +if [ "$1" = '/opt/mssql/bin/sqlservr' ]; then + # If this is the container's first run, initialize the application database + if [ ! -f /tmp/app-initialized ]; then + # Initialize the application database asynchronously in a background process. This allows a) the SQL Server process to be the main process in the container, which allows graceful shutdown and other goodies, and b) us to only start the SQL Server process once, as opposed to starting, stopping, then starting it again. + function initialize_app_database() { + # Wait a bit for SQL Server to start. SQL Server's process doesn't provide a clever way to check if it's up or not, and it needs to be up before we can import the application database + sleep 30s + + echo "RUNNING BUDIBASE SETUP" + + #run the setup script to create the DB and the schema in the DB + /opt/mssql-tools/bin/sqlcmd -S localhost -U sa -P Passw0rd -i setup.sql + + # Note that the container has been initialized so future starts won't wipe changes to the data + touch /tmp/app-initialized + } + initialize_app_database & + fi +fi + +exec "$@" diff --git a/packages/server/scripts/integrations/mssql/data/setup.sql b/packages/server/scripts/integrations/mssql/data/setup.sql new file mode 100644 index 0000000000..f6c94ee2c1 --- /dev/null +++ b/packages/server/scripts/integrations/mssql/data/setup.sql @@ -0,0 +1,34 @@ +USE master; + +IF OBJECT_ID ('dbo.products', 'U') IS NOT NULL + DROP TABLE products; +GO +CREATE TABLE products +( + id int IDENTITY(1,1), + name varchar (20), + description varchar(30) +); +IF OBJECT_ID ('dbo.tasks', 'U') IS NOT NULL + DROP TABLE tasks; +GO +CREATE TABLE tasks +( + taskid int IDENTITY(1,1), + taskname varchar (20) +); + +INSERT products + (name, description) +VALUES + ('Bananas', 'Fruit thing'); + +INSERT products + (name, description) +VALUES + ('Meat', 'Animal thing'); + +INSERT tasks + (taskname) +VALUES + ('Processing'); diff --git a/packages/server/scripts/integrations/mssql/docker-compose.yaml b/packages/server/scripts/integrations/mssql/docker-compose.yaml new file mode 100644 index 0000000000..89222eddaa --- /dev/null +++ b/packages/server/scripts/integrations/mssql/docker-compose.yaml @@ -0,0 +1,12 @@ +version: "3.8" +services: + # password: Passw0rd + # user: sa + # database: master + mssql: + image: bb/mssql + build: + context: . + dockerfile: data/Dockerfile + ports: + - "1433:1433" diff --git a/packages/server/scripts/integrations/mssql/reset.sh b/packages/server/scripts/integrations/mssql/reset.sh new file mode 100755 index 0000000000..32778bd11f --- /dev/null +++ b/packages/server/scripts/integrations/mssql/reset.sh @@ -0,0 +1,3 @@ +#!/bin/bash +docker-compose down +docker volume prune -f From 51dcdf06976628c8ba9c39da30b93373808d5c71 Mon Sep 17 00:00:00 2001 From: mike12345567 Date: Wed, 27 Oct 2021 18:36:27 +0100 Subject: [PATCH 02/12] First version of schema generation for ms-sql, able to retrieve basic tables and rows. --- .../src/integrations/microsoftSqlServer.ts | 103 +++++++++++++++++- 1 file changed, 101 insertions(+), 2 deletions(-) diff --git a/packages/server/src/integrations/microsoftSqlServer.ts b/packages/server/src/integrations/microsoftSqlServer.ts index 2c0108acf4..d97c5f36b6 100644 --- a/packages/server/src/integrations/microsoftSqlServer.ts +++ b/packages/server/src/integrations/microsoftSqlServer.ts @@ -6,10 +6,18 @@ import { SqlQuery, } from "../definitions/datasource" import { getSqlQuery } from "./utils" +import { DatasourcePlus } from "./base/datasourcePlus" +import { Table, TableSchema } from "../definitions/common"; module MSSQLModule { const sqlServer = require("mssql") const Sql = require("./base/sql") + const { FieldTypes } = require("../constants") + const { + buildExternalTableId, + convertType, + finaliseExternalTables, + } = require("./utils") interface MSSQLConfig { user: string @@ -22,6 +30,7 @@ module MSSQLModule { const SCHEMA: Integration = { docs: "https://github.com/tediousjs/node-mssql", + plus: true, description: "Microsoft SQL Server is a relational database management system developed by Microsoft. ", friendlyName: "MS SQL Server", @@ -69,18 +78,66 @@ module MSSQLModule { }, } + // TODO: need to update this + const TYPE_MAP = { + text: FieldTypes.LONGFORM, + blob: FieldTypes.LONGFORM, + enum: FieldTypes.STRING, + varchar: FieldTypes.STRING, + float: FieldTypes.NUMBER, + int: FieldTypes.NUMBER, + numeric: FieldTypes.NUMBER, + bigint: FieldTypes.NUMBER, + mediumint: FieldTypes.NUMBER, + decimal: FieldTypes.NUMBER, + dec: FieldTypes.NUMBER, + double: FieldTypes.NUMBER, + real: FieldTypes.NUMBER, + fixed: FieldTypes.NUMBER, + smallint: FieldTypes.NUMBER, + timestamp: FieldTypes.DATETIME, + date: FieldTypes.DATETIME, + datetime: FieldTypes.DATETIME, + time: FieldTypes.DATETIME, + tinyint: FieldTypes.BOOLEAN, + json: DatasourceFieldTypes.JSON, + } + async function internalQuery(client: any, query: SqlQuery) { try { - return await client.query(query.sql, query.bindings || {}) + if (Array.isArray(query.bindings)) { + let count = 0 + for (let binding of query.bindings) { + client.input(`p${count++}`, binding) + } + } + return await client.query(query.sql) } catch (err) { // @ts-ignore throw new Error(err) } } - class SqlServerIntegration extends Sql { + class SqlServerIntegration extends Sql implements DatasourcePlus { private readonly config: MSSQLConfig static pool: any + public tables: Record = {} + public schemaErrors: Record = {} + + MASTER_TABLES = [ + "spt_fallback_db", + "spt_fallback_dev", + "spt_fallback_usg", + "spt_monitor", + "MSreplication_options" + ] + TABLES_SQL = "SELECT * FROM INFORMATION_SCHEMA.TABLES WHERE TABLE_TYPE='BASE TABLE'" + + getDefinitionSQL(tableName: string) { + return `select * + from INFORMATION_SCHEMA.COLUMNS + where TABLE_NAME='${tableName}'` + } constructor(config: MSSQLConfig) { super("mssql") @@ -89,6 +146,7 @@ module MSSQLModule { ...this.config, options: { encrypt: this.config.encrypt, + enableArithAbort: true, }, } delete clientCfg.encrypt @@ -107,6 +165,46 @@ module MSSQLModule { } } + /** + * Fetches the tables from the sql server database and assigns them to the datasource. + * @param {*} datasourceId - datasourceId to fetch + * @param entities - the tables that are to be built + */ + async buildSchema(datasourceId: string, entities: Record) { + + await this.connect() + let tableNames = await internalQuery(this.client, getSqlQuery(this.TABLES_SQL)) + if (tableNames == null || !Array.isArray(tableNames.recordset)) { + throw "Unable to get list of tables in database" + } + tableNames = tableNames.recordset.map((record: any) => record.TABLE_NAME).filter((name: string) => this.MASTER_TABLES.indexOf(name) === -1) + const tables: Record = {} + for (let tableName of tableNames) { + const definition = await internalQuery(this.client, getSqlQuery(this.getDefinitionSQL(tableName))) + let schema: TableSchema = {} + for (let def of definition.recordset) { + const name = def.COLUMN_NAME + if (typeof name !== "string") { + continue + } + const type: string = convertType(def.DATA_TYPE, TYPE_MAP) + const identity = false + schema[name] = { + autocolumn: identity, + name: name, + type, + } + } + tables[tableName] = { + _id: buildExternalTableId(datasourceId, tableName), + primary: ["id"], + name: tableName, + schema, + } + } + this.tables = tables + } + async read(query: SqlQuery | string) { await this.connect() const response = await internalQuery(this.client, getSqlQuery(query)) @@ -132,6 +230,7 @@ module MSSQLModule { } async query(json: QueryJson) { + await this.connect() const operation = this._operation(json).toLowerCase() const input = this._query(json) const response = await internalQuery(this.client, input) From 974cf7b27ea1193dceb2665601b5f30ba0aa4716 Mon Sep 17 00:00:00 2001 From: mike12345567 Date: Fri, 5 Nov 2021 12:33:48 +0000 Subject: [PATCH 03/12] Linting and updating SQL Server schema generation to include auto column and primary key recognition. --- .../scripts/integrations/mssql/data/setup.sql | 28 ++++++-- .../api/controllers/row/ExternalRequest.ts | 7 +- packages/server/src/integrations/base/sql.ts | 4 +- .../server/src/integrations/base/sqlTable.ts | 44 +++++++++---- .../server/src/integrations/base/utils.ts | 5 +- .../src/integrations/microsoftSqlServer.ts | 66 ++++++++++++++++--- 6 files changed, 123 insertions(+), 31 deletions(-) diff --git a/packages/server/scripts/integrations/mssql/data/setup.sql b/packages/server/scripts/integrations/mssql/data/setup.sql index f6c94ee2c1..766388f46a 100644 --- a/packages/server/scripts/integrations/mssql/data/setup.sql +++ b/packages/server/scripts/integrations/mssql/data/setup.sql @@ -7,15 +7,30 @@ CREATE TABLE products ( id int IDENTITY(1,1), name varchar (20), - description varchar(30) + description varchar(30), + CONSTRAINT pk_products PRIMARY KEY NONCLUSTERED (id) ); + IF OBJECT_ID ('dbo.tasks', 'U') IS NOT NULL DROP TABLE tasks; GO CREATE TABLE tasks ( taskid int IDENTITY(1,1), - taskname varchar (20) + taskname varchar (20), + productid int, + CONSTRAINT pk_tasks PRIMARY KEY NONCLUSTERED (taskid), + CONSTRAINT fk_products FOREIGN KEY (productid) REFERENCES products (id), +); + +IF OBJECT_ID ('dbo.people', 'U') IS NOT NULL + DROP TABLE people; +GO +CREATE TABLE people +( + name varchar(30), + age varchar(20), + CONSTRAINT pk_people PRIMARY KEY NONCLUSTERED (name, age) ); INSERT products @@ -29,6 +44,11 @@ VALUES ('Meat', 'Animal thing'); INSERT tasks - (taskname) + (taskname, productid) VALUES - ('Processing'); + ('Processing', 1); + +INSERT people + (name, age) +VALUES + ('Bob', '30'); diff --git a/packages/server/src/api/controllers/row/ExternalRequest.ts b/packages/server/src/api/controllers/row/ExternalRequest.ts index f538e01f73..23d8deb259 100644 --- a/packages/server/src/api/controllers/row/ExternalRequest.ts +++ b/packages/server/src/api/controllers/row/ExternalRequest.ts @@ -226,7 +226,12 @@ module External { manyRelationships: ManyRelationship[] = [] for (let [key, field] of Object.entries(table.schema)) { // if set already, or not set just skip it - if (row[key] == null || newRow[key] || field.autocolumn || field.type === FieldTypes.FORMULA) { + if ( + row[key] == null || + newRow[key] || + field.autocolumn || + field.type === FieldTypes.FORMULA + ) { continue } // if its an empty string then it means return the column to null (if possible) diff --git a/packages/server/src/integrations/base/sql.ts b/packages/server/src/integrations/base/sql.ts index 738b44afcc..6c64d5c38f 100644 --- a/packages/server/src/integrations/base/sql.ts +++ b/packages/server/src/integrations/base/sql.ts @@ -279,7 +279,9 @@ class SqlQueryBuilder extends SqlTableQueryBuilder { case Operation.DELETE: query = buildDelete(client, json, opts) break - case Operation.CREATE_TABLE: case Operation.UPDATE_TABLE: case Operation.DELETE_TABLE: + case Operation.CREATE_TABLE: + case Operation.UPDATE_TABLE: + case Operation.DELETE_TABLE: return this._tableQuery(json) default: throw `Operation type is not supported by SQL query builder` diff --git a/packages/server/src/integrations/base/sqlTable.ts b/packages/server/src/integrations/base/sqlTable.ts index e5249dfe7c..974f395063 100644 --- a/packages/server/src/integrations/base/sqlTable.ts +++ b/packages/server/src/integrations/base/sqlTable.ts @@ -6,7 +6,12 @@ import SchemaBuilder = Knex.SchemaBuilder import CreateTableBuilder = Knex.CreateTableBuilder const { FieldTypes, RelationshipTypes } = require("../../constants") -function generateSchema(schema: CreateTableBuilder, table: Table, tables: Record, oldTable: null | Table = null) { +function generateSchema( + schema: CreateTableBuilder, + table: Table, + tables: Record, + oldTable: null | Table = null +) { let primaryKey = table && table.primary ? table.primary[0] : null const columns = Object.values(table.schema) // all columns in a junction table will be meta @@ -19,17 +24,21 @@ function generateSchema(schema: CreateTableBuilder, table: Table, tables: Record schema.primary(metaCols.map(col => col.name)) } - // check if any columns need added const foreignKeys = Object.values(table.schema).map(col => col.foreignKey) for (let [key, column] of Object.entries(table.schema)) { // skip things that are already correct const oldColumn = oldTable ? oldTable.schema[key] : null - if ((oldColumn && oldColumn.type === column.type) || (primaryKey === key && !isJunction)) { + if ( + (oldColumn && oldColumn.type === column.type) || + (primaryKey === key && !isJunction) + ) { continue } switch (column.type) { - case FieldTypes.STRING: case FieldTypes.OPTIONS: case FieldTypes.LONGFORM: + case FieldTypes.STRING: + case FieldTypes.OPTIONS: + case FieldTypes.LONGFORM: schema.string(key) break case FieldTypes.NUMBER: @@ -67,7 +76,9 @@ function generateSchema(schema: CreateTableBuilder, table: Table, tables: Record throw "Referenced table doesn't exist" } schema.integer(column.foreignKey).unsigned() - schema.foreign(column.foreignKey).references(`${tableName}.${relatedTable.primary[0]}`) + schema + .foreign(column.foreignKey) + .references(`${tableName}.${relatedTable.primary[0]}`) } break } @@ -76,7 +87,10 @@ function generateSchema(schema: CreateTableBuilder, table: Table, tables: Record // need to check if any columns have been deleted if (oldTable) { const deletedColumns = Object.entries(oldTable.schema) - .filter(([key, schema]) => schema.type !== FieldTypes.LINK && table.schema[key] == null) + .filter( + ([key, schema]) => + schema.type !== FieldTypes.LINK && table.schema[key] == null + ) .map(([key]) => key) deletedColumns.forEach(key => { if (oldTable.constrained && oldTable.constrained.indexOf(key) !== -1) { @@ -92,7 +106,7 @@ function generateSchema(schema: CreateTableBuilder, table: Table, tables: Record function buildCreateTable( knex: Knex, table: Table, - tables: Record, + tables: Record ): SchemaBuilder { return knex.schema.createTable(table.name, schema => { generateSchema(schema, table, tables) @@ -103,17 +117,14 @@ function buildUpdateTable( knex: Knex, table: Table, tables: Record, - oldTable: Table, + oldTable: Table ): SchemaBuilder { return knex.schema.alterTable(table.name, schema => { generateSchema(schema, table, tables, oldTable) }) } -function buildDeleteTable( - knex: Knex, - table: Table, -): SchemaBuilder { +function buildDeleteTable(knex: Knex, table: Table): SchemaBuilder { return knex.schema.dropTable(table.name) } @@ -151,7 +162,12 @@ class SqlTableQueryBuilder { if (!json.meta || !json.meta.table) { throw "Must specify old table for update" } - query = buildUpdateTable(client, json.table, json.meta.tables, json.meta.table) + query = buildUpdateTable( + client, + json.table, + json.meta.tables, + json.meta.table + ) break case Operation.DELETE_TABLE: query = buildDeleteTable(client, json.table) @@ -164,4 +180,4 @@ class SqlTableQueryBuilder { } export default SqlTableQueryBuilder -module.exports = SqlTableQueryBuilder \ No newline at end of file +module.exports = SqlTableQueryBuilder diff --git a/packages/server/src/integrations/base/utils.ts b/packages/server/src/integrations/base/utils.ts index 5757232bc7..086912b920 100644 --- a/packages/server/src/integrations/base/utils.ts +++ b/packages/server/src/integrations/base/utils.ts @@ -4,7 +4,10 @@ import { Datasource } from "../../definitions/common" module DatasourceUtils { const { integrations } = require("../index") - export async function makeExternalQuery(datasource: Datasource, json: QueryJson) { + export async function makeExternalQuery( + datasource: Datasource, + json: QueryJson + ) { const Integration = integrations[datasource.source] // query is the opinionated function if (Integration.prototype.query) { diff --git a/packages/server/src/integrations/microsoftSqlServer.ts b/packages/server/src/integrations/microsoftSqlServer.ts index d97c5f36b6..f8fd638082 100644 --- a/packages/server/src/integrations/microsoftSqlServer.ts +++ b/packages/server/src/integrations/microsoftSqlServer.ts @@ -7,7 +7,7 @@ import { } from "../definitions/datasource" import { getSqlQuery } from "./utils" import { DatasourcePlus } from "./base/datasourcePlus" -import { Table, TableSchema } from "../definitions/common"; +import { Table, TableSchema } from "../definitions/common" module MSSQLModule { const sqlServer = require("mssql") @@ -129,9 +129,10 @@ module MSSQLModule { "spt_fallback_dev", "spt_fallback_usg", "spt_monitor", - "MSreplication_options" + "MSreplication_options", ] - TABLES_SQL = "SELECT * FROM INFORMATION_SCHEMA.TABLES WHERE TABLE_TYPE='BASE TABLE'" + TABLES_SQL = + "SELECT * FROM INFORMATION_SCHEMA.TABLES WHERE TABLE_TYPE='BASE TABLE'" getDefinitionSQL(tableName: string) { return `select * @@ -139,6 +140,28 @@ module MSSQLModule { where TABLE_NAME='${tableName}'` } + getConstraintsSQL(tableName: string) { + return `SELECT * FROM INFORMATION_SCHEMA.TABLE_CONSTRAINTS AS TC + INNER JOIN INFORMATION_SCHEMA.KEY_COLUMN_USAGE AS KU + ON TC.CONSTRAINT_TYPE = 'PRIMARY KEY' + AND TC.CONSTRAINT_NAME = KU.CONSTRAINT_NAME + AND KU.table_name='${tableName}' + ORDER BY + KU.TABLE_NAME, + KU.ORDINAL_POSITION;` + } + + getAutoColumnsSQL(tableName: string) { + return `SELECT + COLUMNPROPERTY(OBJECT_ID(TABLE_SCHEMA+'.'+TABLE_NAME),COLUMN_NAME,'IsComputed') + AS IS_COMPUTED, + COLUMNPROPERTY(object_id(TABLE_SCHEMA+'.'+TABLE_NAME), COLUMN_NAME, 'IsIdentity') + AS IS_IDENTITY, + * + FROM INFORMATION_SCHEMA.COLUMNS + WHERE TABLE_NAME='${tableName}'` + } + constructor(config: MSSQLConfig) { super("mssql") this.config = config @@ -171,16 +194,39 @@ module MSSQLModule { * @param entities - the tables that are to be built */ async buildSchema(datasourceId: string, entities: Record) { - await this.connect() - let tableNames = await internalQuery(this.client, getSqlQuery(this.TABLES_SQL)) + let tableNames = await internalQuery( + this.client, + getSqlQuery(this.TABLES_SQL) + ) if (tableNames == null || !Array.isArray(tableNames.recordset)) { throw "Unable to get list of tables in database" } - tableNames = tableNames.recordset.map((record: any) => record.TABLE_NAME).filter((name: string) => this.MASTER_TABLES.indexOf(name) === -1) + tableNames = tableNames.recordset + .map((record: any) => record.TABLE_NAME) + .filter((name: string) => this.MASTER_TABLES.indexOf(name) === -1) const tables: Record = {} for (let tableName of tableNames) { - const definition = await internalQuery(this.client, getSqlQuery(this.getDefinitionSQL(tableName))) + const definition = await internalQuery( + this.client, + getSqlQuery(this.getDefinitionSQL(tableName)) + ) + const constraints = await internalQuery( + this.client, + getSqlQuery(this.getConstraintsSQL(tableName)) + ) + const columns = await internalQuery( + this.client, + getSqlQuery(this.getAutoColumnsSQL(tableName)) + ) + const autoColumns = columns.recordset + .filter((col: any) => col.IS_COMPUTED || col.IS_IDENTITY) + .map((col: any) => col.COLUMN_NAME) + const primaryKeys = constraints.recordset + .filter( + (constraint: any) => constraint.CONSTRAINT_TYPE === "PRIMARY KEY" + ) + .map((constraint: any) => constraint.COLUMN_NAME) let schema: TableSchema = {} for (let def of definition.recordset) { const name = def.COLUMN_NAME @@ -188,16 +234,16 @@ module MSSQLModule { continue } const type: string = convertType(def.DATA_TYPE, TYPE_MAP) - const identity = false + schema[name] = { - autocolumn: identity, + autocolumn: !!autoColumns.find((col: string) => col === name), name: name, type, } } tables[tableName] = { _id: buildExternalTableId(datasourceId, tableName), - primary: ["id"], + primary: primaryKeys, name: tableName, schema, } From ed89efba70cb3e58c8d88f54eaa91dd32d12b904 Mon Sep 17 00:00:00 2001 From: mike12345567 Date: Fri, 5 Nov 2021 12:41:26 +0000 Subject: [PATCH 04/12] Cleaning up repeated work in sql server building of schema. --- .../src/integrations/microsoftSqlServer.ts | 39 +++++++++---------- 1 file changed, 18 insertions(+), 21 deletions(-) diff --git a/packages/server/src/integrations/microsoftSqlServer.ts b/packages/server/src/integrations/microsoftSqlServer.ts index f8fd638082..7071ce6f75 100644 --- a/packages/server/src/integrations/microsoftSqlServer.ts +++ b/packages/server/src/integrations/microsoftSqlServer.ts @@ -188,6 +188,10 @@ module MSSQLModule { } } + async runSQL(sql: string) { + return (await internalQuery(this.client, getSqlQuery(sql))).recordset + } + /** * Fetches the tables from the sql server database and assigns them to the datasource. * @param {*} datasourceId - datasourceId to fetch @@ -195,40 +199,33 @@ module MSSQLModule { */ async buildSchema(datasourceId: string, entities: Record) { await this.connect() - let tableNames = await internalQuery( - this.client, - getSqlQuery(this.TABLES_SQL) - ) + let tableNames = await this.runSQL(this.TABLES_SQL) if (tableNames == null || !Array.isArray(tableNames.recordset)) { throw "Unable to get list of tables in database" } tableNames = tableNames.recordset .map((record: any) => record.TABLE_NAME) .filter((name: string) => this.MASTER_TABLES.indexOf(name) === -1) + const tables: Record = {} for (let tableName of tableNames) { - const definition = await internalQuery( - this.client, - getSqlQuery(this.getDefinitionSQL(tableName)) - ) - const constraints = await internalQuery( - this.client, - getSqlQuery(this.getConstraintsSQL(tableName)) - ) - const columns = await internalQuery( - this.client, - getSqlQuery(this.getAutoColumnsSQL(tableName)) - ) - const autoColumns = columns.recordset - .filter((col: any) => col.IS_COMPUTED || col.IS_IDENTITY) - .map((col: any) => col.COLUMN_NAME) - const primaryKeys = constraints.recordset + // get the column definition (type) + const definition = await this.runSQL(this.getDefinitionSQL(tableName)) + // find primary key constraints + const constraints = await this.runSQL(this.getConstraintsSQL(tableName)) + // find the computed and identity columns (auto columns) + const columns = await this.runSQL(this.getAutoColumnsSQL(tableName)) + const primaryKeys = constraints .filter( (constraint: any) => constraint.CONSTRAINT_TYPE === "PRIMARY KEY" ) .map((constraint: any) => constraint.COLUMN_NAME) + const autoColumns = columns + .filter((col: any) => col.IS_COMPUTED || col.IS_IDENTITY) + .map((col: any) => col.COLUMN_NAME) + let schema: TableSchema = {} - for (let def of definition.recordset) { + for (let def of definition) { const name = def.COLUMN_NAME if (typeof name !== "string") { continue From 948ec067d571e49a47bb8a844677df9f4b14fe97 Mon Sep 17 00:00:00 2001 From: mike12345567 Date: Fri, 5 Nov 2021 13:48:13 +0000 Subject: [PATCH 05/12] Updating underlying sql to not use ilike unless in postgres client. --- .../api/controllers/row/ExternalRequest.ts | 2 +- packages/server/src/integrations/base/sql.ts | 421 +++++++++--------- .../src/integrations/microsoftSqlServer.ts | 4 +- 3 files changed, 220 insertions(+), 207 deletions(-) diff --git a/packages/server/src/api/controllers/row/ExternalRequest.ts b/packages/server/src/api/controllers/row/ExternalRequest.ts index 23d8deb259..d4e8d475a2 100644 --- a/packages/server/src/api/controllers/row/ExternalRequest.ts +++ b/packages/server/src/api/controllers/row/ExternalRequest.ts @@ -342,7 +342,7 @@ module External { table: Table, relationships: RelationshipsJson[] ) { - if (rows[0].read === true) { + if (!rows || rows.length === 0 || rows[0].read === true) { return [] } let finalRows: { [key: string]: Row } = {} diff --git a/packages/server/src/integrations/base/sql.ts b/packages/server/src/integrations/base/sql.ts index 6c64d5c38f..06a9d0aa10 100644 --- a/packages/server/src/integrations/base/sql.ts +++ b/packages/server/src/integrations/base/sql.ts @@ -29,222 +29,232 @@ function parseBody(body: any) { return body } -// right now we only do filters on the specific table being queried -function addFilters( - tableName: string, - query: KnexQuery, - filters: SearchFilters | undefined -): KnexQuery { - function iterate( - structure: { [key: string]: any }, - fn: (key: string, value: any) => void - ) { - for (let [key, value] of Object.entries(structure)) { - fn(`${tableName}.${key}`, value) - } +class InternalBuilder { + private readonly client: string + + constructor(client: string) { + this.client = client } - if (!filters) { - return query - } - // if all or specified in filters, then everything is an or - const allOr = filters.allOr - if (filters.oneOf) { - iterate(filters.oneOf, (key, array) => { - const fnc = allOr ? "orWhereIn" : "whereIn" - query = query[fnc](key, array) - }) - } - if (filters.string) { - iterate(filters.string, (key, value) => { - const fnc = allOr ? "orWhere" : "where" - query = query[fnc](key, "ilike", `${value}%`) - }) - } - if (filters.fuzzy) { - iterate(filters.fuzzy, (key, value) => { - const fnc = allOr ? "orWhere" : "where" - query = query[fnc](key, "ilike", `%${value}%`) - }) - } - if (filters.range) { - iterate(filters.range, (key, value) => { - if (!value.high || !value.low) { - return + + // right now we only do filters on the specific table being queried + addFilters( + tableName: string, + query: KnexQuery, + filters: SearchFilters | undefined + ): KnexQuery { + function iterate( + structure: { [key: string]: any }, + fn: (key: string, value: any) => void + ) { + for (let [key, value] of Object.entries(structure)) { + fn(`${tableName}.${key}`, value) } - const fnc = allOr ? "orWhereBetween" : "whereBetween" - query = query[fnc](key, [value.low, value.high]) - }) - } - if (filters.equal) { - iterate(filters.equal, (key, value) => { - const fnc = allOr ? "orWhere" : "where" - query = query[fnc]({ [key]: value }) - }) - } - if (filters.notEqual) { - iterate(filters.notEqual, (key, value) => { - const fnc = allOr ? "orWhereNot" : "whereNot" - query = query[fnc]({ [key]: value }) - }) - } - if (filters.empty) { - iterate(filters.empty, key => { - const fnc = allOr ? "orWhereNull" : "whereNull" - query = query[fnc](key) - }) - } - if (filters.notEmpty) { - iterate(filters.notEmpty, key => { - const fnc = allOr ? "orWhereNotNull" : "whereNotNull" - query = query[fnc](key) - }) - } - return query -} - -function addRelationships( - knex: Knex, - query: KnexQuery, - fields: string | string[], - fromTable: string, - relationships: RelationshipsJson[] | undefined -): KnexQuery { - if (!relationships) { + } + if (!filters) { + return query + } + // if all or specified in filters, then everything is an or + const allOr = filters.allOr + if (filters.oneOf) { + iterate(filters.oneOf, (key, array) => { + const fnc = allOr ? "orWhereIn" : "whereIn" + query = query[fnc](key, array) + }) + } + if (filters.string) { + iterate(filters.string, (key, value) => { + const fnc = allOr ? "orWhere" : "where" + // postgres supports ilike, nothing else does + if (this.client === "pg") { + query = query[fnc](key, "ilike", `${value}%`) + } else { + const rawFnc = `${fnc}Raw` + // @ts-ignore + query = query[rawFnc](`LOWER(${key}) LIKE ?`, [`${value}%`]) + } + }) + } + if (filters.fuzzy) { + iterate(filters.fuzzy, (key, value) => { + const fnc = allOr ? "orWhere" : "where" + // postgres supports ilike, nothing else does + if (this.client === "pg") { + query = query[fnc](key, "ilike", `%${value}%`) + } else { + const rawFnc = `${fnc}Raw` + // @ts-ignore + query = query[rawFnc](`LOWER(${key}) LIKE ?`, [`%${value}%`]) + } + }) + } + if (filters.range) { + iterate(filters.range, (key, value) => { + if (!value.high || !value.low) { + return + } + const fnc = allOr ? "orWhereBetween" : "whereBetween" + query = query[fnc](key, [value.low, value.high]) + }) + } + if (filters.equal) { + iterate(filters.equal, (key, value) => { + const fnc = allOr ? "orWhere" : "where" + query = query[fnc]({ [key]: value }) + }) + } + if (filters.notEqual) { + iterate(filters.notEqual, (key, value) => { + const fnc = allOr ? "orWhereNot" : "whereNot" + query = query[fnc]({ [key]: value }) + }) + } + if (filters.empty) { + iterate(filters.empty, key => { + const fnc = allOr ? "orWhereNull" : "whereNull" + query = query[fnc](key) + }) + } + if (filters.notEmpty) { + iterate(filters.notEmpty, key => { + const fnc = allOr ? "orWhereNotNull" : "whereNotNull" + query = query[fnc](key) + }) + } return query } - for (let relationship of relationships) { - const from = relationship.from, - to = relationship.to, - toTable = relationship.tableName - if (!relationship.through) { - // @ts-ignore - query = query.leftJoin( - toTable, - `${fromTable}.${from}`, - `${toTable}.${to}` - ) - } else { - const throughTable = relationship.through - const fromPrimary = relationship.fromPrimary - const toPrimary = relationship.toPrimary - query = query + + addRelationships( + knex: Knex, + query: KnexQuery, + fields: string | string[], + fromTable: string, + relationships: RelationshipsJson[] | undefined + ): KnexQuery { + if (!relationships) { + return query + } + for (let relationship of relationships) { + const from = relationship.from, + to = relationship.to, + toTable = relationship.tableName + if (!relationship.through) { // @ts-ignore - .leftJoin( - throughTable, - `${fromTable}.${fromPrimary}`, - `${throughTable}.${from}` + query = query.leftJoin( + toTable, + `${fromTable}.${from}`, + `${toTable}.${to}` ) - .leftJoin(toTable, `${toTable}.${toPrimary}`, `${throughTable}.${to}`) + } else { + const throughTable = relationship.through + const fromPrimary = relationship.fromPrimary + const toPrimary = relationship.toPrimary + query = query + // @ts-ignore + .leftJoin( + throughTable, + `${fromTable}.${fromPrimary}`, + `${throughTable}.${from}` + ) + .leftJoin(toTable, `${toTable}.${toPrimary}`, `${throughTable}.${to}`) + } + } + return query.limit(BASE_LIMIT) + } + + create(knex: Knex, json: QueryJson, opts: QueryOptions): KnexQuery { + const { endpoint, body } = json + let query: KnexQuery = knex(endpoint.entityId) + const parsedBody = parseBody(body) + // make sure no null values in body for creation + for (let [key, value] of Object.entries(parsedBody)) { + if (value == null) { + delete parsedBody[key] + } + } + // mysql can't use returning + if (opts.disableReturning) { + return query.insert(parsedBody) + } else { + return query.insert(parsedBody).returning("*") } } - return query.limit(BASE_LIMIT) -} -function buildCreate( - knex: Knex, - json: QueryJson, - opts: QueryOptions -): KnexQuery { - const { endpoint, body } = json - let query: KnexQuery = knex(endpoint.entityId) - const parsedBody = parseBody(body) - // make sure no null values in body for creation - for (let [key, value] of Object.entries(parsedBody)) { - if (value == null) { - delete parsedBody[key] + read(knex: Knex, json: QueryJson, limit: number): KnexQuery { + let { endpoint, resource, filters, sort, paginate, relationships } = json + const tableName = endpoint.entityId + // select all if not specified + if (!resource) { + resource = { fields: [] } } - } - // mysql can't use returning - if (opts.disableReturning) { - return query.insert(parsedBody) - } else { - return query.insert(parsedBody).returning("*") - } -} - -function buildRead(knex: Knex, json: QueryJson, limit: number): KnexQuery { - let { endpoint, resource, filters, sort, paginate, relationships } = json - const tableName = endpoint.entityId - // select all if not specified - if (!resource) { - resource = { fields: [] } - } - let selectStatement: string | string[] = "*" - // handle select - if (resource.fields && resource.fields.length > 0) { - // select the resources as the format "table.columnName" - this is what is provided - // by the resource builder further up - selectStatement = resource.fields.map(field => `${field} as ${field}`) - } - let foundLimit = limit || BASE_LIMIT - // handle pagination - let foundOffset: number | null = null - if (paginate && paginate.page && paginate.limit) { + let selectStatement: string | string[] = "*" + // handle select + if (resource.fields && resource.fields.length > 0) { + // select the resources as the format "table.columnName" - this is what is provided + // by the resource builder further up + selectStatement = resource.fields.map(field => `${field} as ${field}`) + } + let foundLimit = limit || BASE_LIMIT + // handle pagination + let foundOffset: number | null = null + if (paginate && paginate.page && paginate.limit) { + // @ts-ignore + const page = paginate.page <= 1 ? 0 : paginate.page - 1 + const offset = page * paginate.limit + foundLimit = paginate.limit + foundOffset = offset + } else if (paginate && paginate.limit) { + foundLimit = paginate.limit + } + // start building the query + let query: KnexQuery = knex(tableName).limit(foundLimit) + if (foundOffset) { + query = query.offset(foundOffset) + } + if (sort) { + for (let [key, value] of Object.entries(sort)) { + const direction = value === SortDirection.ASCENDING ? "asc" : "desc" + query = query.orderBy(key, direction) + } + } + query = this.addFilters(tableName, query, filters) // @ts-ignore - const page = paginate.page <= 1 ? 0 : paginate.page - 1 - const offset = page * paginate.limit - foundLimit = paginate.limit - foundOffset = offset - } else if (paginate && paginate.limit) { - foundLimit = paginate.limit + let preQuery: KnexQuery = knex({ + // @ts-ignore + [tableName]: query, + }).select(selectStatement) + // handle joins + return this.addRelationships( + knex, + preQuery, + selectStatement, + tableName, + relationships + ) } - // start building the query - let query: KnexQuery = knex(tableName).limit(foundLimit) - if (foundOffset) { - query = query.offset(foundOffset) - } - if (sort) { - for (let [key, value] of Object.entries(sort)) { - const direction = value === SortDirection.ASCENDING ? "asc" : "desc" - query = query.orderBy(key, direction) + + update(knex: Knex, json: QueryJson, opts: QueryOptions): KnexQuery { + const { endpoint, body, filters } = json + let query: KnexQuery = knex(endpoint.entityId) + const parsedBody = parseBody(body) + query = this.addFilters(endpoint.entityId, query, filters) + // mysql can't use returning + if (opts.disableReturning) { + return query.update(parsedBody) + } else { + return query.update(parsedBody).returning("*") } } - query = addFilters(tableName, query, filters) - // @ts-ignore - let preQuery: KnexQuery = knex({ - // @ts-ignore - [tableName]: query, - }).select(selectStatement) - // handle joins - return addRelationships( - knex, - preQuery, - selectStatement, - tableName, - relationships - ) -} -function buildUpdate( - knex: Knex, - json: QueryJson, - opts: QueryOptions -): KnexQuery { - const { endpoint, body, filters } = json - let query: KnexQuery = knex(endpoint.entityId) - const parsedBody = parseBody(body) - query = addFilters(endpoint.entityId, query, filters) - // mysql can't use returning - if (opts.disableReturning) { - return query.update(parsedBody) - } else { - return query.update(parsedBody).returning("*") - } -} - -function buildDelete( - knex: Knex, - json: QueryJson, - opts: QueryOptions -): KnexQuery { - const { endpoint, filters } = json - let query: KnexQuery = knex(endpoint.entityId) - query = addFilters(endpoint.entityId, query, filters) - // mysql can't use returning - if (opts.disableReturning) { - return query.delete() - } else { - return query.delete().returning("*") + delete(knex: Knex, json: QueryJson, opts: QueryOptions): KnexQuery { + const { endpoint, filters } = json + let query: KnexQuery = knex(endpoint.entityId) + query = this.addFilters(endpoint.entityId, query, filters) + // mysql can't use returning + if (opts.disableReturning) { + return query.delete() + } else { + return query.delete().returning("*") + } } } @@ -266,18 +276,19 @@ class SqlQueryBuilder extends SqlTableQueryBuilder { const sqlClient = this.getSqlClient() const client = knex({ client: sqlClient }) let query + const builder = new InternalBuilder(sqlClient) switch (this._operation(json)) { case Operation.CREATE: - query = buildCreate(client, json, opts) + query = builder.create(client, json, opts) break case Operation.READ: - query = buildRead(client, json, this.limit) + query = builder.read(client, json, this.limit) break case Operation.UPDATE: - query = buildUpdate(client, json, opts) + query = builder.update(client, json, opts) break case Operation.DELETE: - query = buildDelete(client, json, opts) + query = builder.delete(client, json, opts) break case Operation.CREATE_TABLE: case Operation.UPDATE_TABLE: diff --git a/packages/server/src/integrations/microsoftSqlServer.ts b/packages/server/src/integrations/microsoftSqlServer.ts index 7071ce6f75..bcb22f50a9 100644 --- a/packages/server/src/integrations/microsoftSqlServer.ts +++ b/packages/server/src/integrations/microsoftSqlServer.ts @@ -245,7 +245,9 @@ module MSSQLModule { schema, } } - this.tables = tables + const final = finaliseExternalTables(tables) + this.tables = final.tables + this.schemaErrors = final.errors } async read(query: SqlQuery | string) { From c22356fb4d3378a2225dc6e1ab491401a3ed2f5a Mon Sep 17 00:00:00 2001 From: mike12345567 Date: Fri, 5 Nov 2021 18:55:36 +0000 Subject: [PATCH 06/12] Fixing an issue with relationship modal breaking when multiple data sources available to relate to, also fixing an pile of issues with creating and reading rows from SQL server plus. --- .../DataTable/modals/CreateEditColumn.svelte | 13 ++- packages/server/src/definitions/datasource.ts | 2 +- packages/server/src/integrations/base/sql.ts | 83 +++++++++++++++++++ .../src/integrations/microsoftSqlServer.ts | 35 +++++--- packages/server/src/integrations/mysql.ts | 61 +------------- packages/server/src/integrations/utils.ts | 2 +- 6 files changed, 121 insertions(+), 75 deletions(-) diff --git a/packages/builder/src/components/backend/DataTable/modals/CreateEditColumn.svelte b/packages/builder/src/components/backend/DataTable/modals/CreateEditColumn.svelte index ebfea9cee6..6ee0c48d2a 100644 --- a/packages/builder/src/components/backend/DataTable/modals/CreateEditColumn.svelte +++ b/packages/builder/src/components/backend/DataTable/modals/CreateEditColumn.svelte @@ -59,9 +59,6 @@ let deletion $: checkConstraints(field) - $: tableOptions = $tables.list.filter( - opt => opt._id !== $tables.draft._id && opt.type === table.type - ) $: required = !!field?.constraints?.presence || primaryDisplay $: uneditable = $tables.selected?._id === TableNames.USERS && @@ -88,6 +85,14 @@ field.type !== LINK_TYPE && !uneditable && field.type !== AUTO_TYPE $: relationshipOptions = getRelationshipOptions(field) $: external = table.type === "external" + // in the case of internal tables the sourceId will just be undefined + $: tableOptions = $tables.list.filter( + opt => + opt._id !== $tables.draft._id && + opt.type === table.type && + table.sourceId === opt.sourceId + ) + $: console.log(tableOptions) async function saveColumn() { if (field.type === AUTO_TYPE) { @@ -174,7 +179,7 @@ if (!field || !field.tableId) { return null } - const linkTable = tableOptions.find(table => table._id === field.tableId) + const linkTable = tableOptions?.find(table => table._id === field.tableId) if (!linkTable) { return null } diff --git a/packages/server/src/definitions/datasource.ts b/packages/server/src/definitions/datasource.ts index a82e50b140..6e6cb02f4f 100644 --- a/packages/server/src/definitions/datasource.ts +++ b/packages/server/src/definitions/datasource.ts @@ -119,7 +119,7 @@ export interface SortJson { export interface PaginationJson { limit: number - page: string | number + page?: string | number } export interface RelationshipsJson { diff --git a/packages/server/src/integrations/base/sql.ts b/packages/server/src/integrations/base/sql.ts index 06a9d0aa10..d72f87958b 100644 --- a/packages/server/src/integrations/base/sql.ts +++ b/packages/server/src/integrations/base/sql.ts @@ -216,6 +216,10 @@ class InternalBuilder { query = query.orderBy(key, direction) } } + if (this.client === "mssql" && !sort && paginate?.limit) { + // @ts-ignore + query = query.orderBy(json.meta?.table?.primary[0]) + } query = this.addFilters(tableName, query, filters) // @ts-ignore let preQuery: KnexQuery = knex({ @@ -301,6 +305,85 @@ class SqlQueryBuilder extends SqlTableQueryBuilder { // @ts-ignore return query.toSQL().toNative() } + + async getReturningRow(queryFn: Function, json: QueryJson) { + if (!json.extra || !json.extra.idFilter) { + return {} + } + const input = this._query({ + endpoint: { + ...json.endpoint, + operation: Operation.READ, + }, + resource: { + fields: [], + }, + filters: json.extra.idFilter, + paginate: { + limit: 1, + }, + meta: json.meta, + }) + return queryFn(input, Operation.READ) + } + + // when creating if an ID has been inserted need to make sure + // the id filter is enriched with it before trying to retrieve the row + checkLookupKeys(id: any, json: QueryJson) { + if (!id || !json.meta?.table || !json.meta.table.primary) { + return json + } + const primaryKey = json.meta.table.primary?.[0] + json.extra = { + idFilter: { + equal: { + [primaryKey]: id, + }, + }, + } + return json + } + + // this function recreates the returning functionality of postgres + async queryWithReturning( + json: QueryJson, + queryFn: Function, + processFn: Function = (result: any) => result + ) { + const sqlClient = this.getSqlClient() + const operation = this._operation(json) + const input = this._query(json, { disableReturning: true }) + if (Array.isArray(input)) { + const responses = [] + for (let query of input) { + responses.push(await queryFn(query, operation)) + } + return responses + } + let row + // need to manage returning, a feature mySQL can't do + if (operation === Operation.DELETE) { + row = processFn(await this.getReturningRow(queryFn, json)) + } + const response = await queryFn(input, operation) + const results = processFn(response) + // same as delete, manage returning + if (operation === Operation.CREATE || operation === Operation.UPDATE) { + let id + if (sqlClient === "mssql") { + id = results?.[0].id + } else if (sqlClient === "mysql") { + id = results?.insertId + } + row = processFn( + await this.getReturningRow(queryFn, this.checkLookupKeys(id, json)) + ) + } + if (operation !== Operation.READ) { + return row + } + return results.length ? results : [{ [operation.toLowerCase()]: true }] + } } module.exports = SqlQueryBuilder diff --git a/packages/server/src/integrations/microsoftSqlServer.ts b/packages/server/src/integrations/microsoftSqlServer.ts index bcb22f50a9..1ee64759ef 100644 --- a/packages/server/src/integrations/microsoftSqlServer.ts +++ b/packages/server/src/integrations/microsoftSqlServer.ts @@ -1,8 +1,9 @@ import { - Integration, DatasourceFieldTypes, - QueryTypes, + Integration, + Operation, QueryJson, + QueryTypes, SqlQuery, } from "../definitions/datasource" import { getSqlQuery } from "./utils" @@ -103,15 +104,26 @@ module MSSQLModule { json: DatasourceFieldTypes.JSON, } - async function internalQuery(client: any, query: SqlQuery) { + async function internalQuery( + client: any, + query: SqlQuery, + operation: string | undefined = undefined + ) { + const request = client.request() try { if (Array.isArray(query.bindings)) { let count = 0 for (let binding of query.bindings) { - client.input(`p${count++}`, binding) + request.input(`p${count++}`, binding) } } - return await client.query(query.sql) + // this is a hack to get the inserted ID back, + // no way to do this with Knex nicely + const sql = + operation === Operation.CREATE + ? `${query.sql}; SELECT SCOPE_IDENTITY() AS id;` + : query.sql + return await request.query(sql) } catch (err) { // @ts-ignore throw new Error(err) @@ -180,8 +192,7 @@ module MSSQLModule { async connect() { try { - const client = await this.pool.connect() - this.client = client.request() + this.client = await this.pool.connect() } catch (err) { // @ts-ignore throw new Error(err) @@ -276,10 +287,12 @@ module MSSQLModule { async query(json: QueryJson) { await this.connect() - const operation = this._operation(json).toLowerCase() - const input = this._query(json) - const response = await internalQuery(this.client, input) - return response.recordset ? response.recordset : [{ [operation]: true }] + const operation = this._operation(json) + const queryFn = (query: any, op: string) => + internalQuery(this.client, query, op) + const processFn = (result: any) => + result.recordset ? result.recordset : [{ [operation]: true }] + return this.queryWithReturning(json, queryFn, processFn) } } diff --git a/packages/server/src/integrations/mysql.ts b/packages/server/src/integrations/mysql.ts index 30cbd836b5..ad313d9302 100644 --- a/packages/server/src/integrations/mysql.ts +++ b/packages/server/src/integrations/mysql.ts @@ -223,67 +223,12 @@ module MySQLModule { return results.length ? results : [{ deleted: true }] } - async getReturningRow(json: QueryJson) { - if (!json.extra || !json.extra.idFilter) { - return {} - } - const input = this._query({ - endpoint: { - ...json.endpoint, - operation: Operation.READ, - }, - fields: [], - filters: json.extra.idFilter, - paginate: { - limit: 1, - }, - }) - return internalQuery(this.client, input, false) - } - - // when creating if an ID has been inserted need to make sure - // the id filter is enriched with it before trying to retrieve the row - checkLookupKeys(results: any, json: QueryJson) { - if (!results?.insertId || !json.meta?.table || !json.meta.table.primary) { - return json - } - const primaryKey = json.meta.table.primary?.[0] - json.extra = { - idFilter: { - equal: { - [primaryKey]: results.insertId, - }, - }, - } - return json - } - async query(json: QueryJson) { - const operation = this._operation(json) this.client.connect() - const input = this._query(json, { disableReturning: true }) - if (Array.isArray(input)) { - const responses = [] - for (let query of input) { - responses.push(await internalQuery(this.client, query, false)) - } - return responses - } - let row - // need to manage returning, a feature mySQL can't do - if (operation === operation.DELETE) { - row = this.getReturningRow(json) - } - const results = await internalQuery(this.client, input, false) - // same as delete, manage returning - if (operation === Operation.CREATE || operation === Operation.UPDATE) { - row = this.getReturningRow(this.checkLookupKeys(results, json)) - } + const queryFn = (query: any) => internalQuery(this.client, query, false) + const output = await this.queryWithReturning(json, queryFn) this.client.end() - if (operation !== Operation.READ) { - return row - } - return results.length ? results : [{ [operation.toLowerCase()]: true }] + return output } } diff --git a/packages/server/src/integrations/utils.ts b/packages/server/src/integrations/utils.ts index f91d25423c..55ec086b7e 100644 --- a/packages/server/src/integrations/utils.ts +++ b/packages/server/src/integrations/utils.ts @@ -1,4 +1,4 @@ -import { SqlQuery } from "../definitions/datasource" +import { Operation, SqlQuery } from "../definitions/datasource" import { Datasource, Table } from "../definitions/common" import { SourceNames } from "../definitions/datasource" const { DocumentTypes, SEPARATOR } = require("../db/utils") From 15907280b2dca89e04b40a58b051f78b6f15cb51 Mon Sep 17 00:00:00 2001 From: mike12345567 Date: Mon, 8 Nov 2021 17:25:05 +0000 Subject: [PATCH 07/12] Fixing #3182 so that sorting is disabled for certain column types that we cannot use, as well as fixing some issues with MS-SQL plus tables. --- .../components/backend/DataTable/Table.svelte | 15 +++++++++++++- .../DataTable/modals/CreateEditColumn.svelte | 1 - packages/builder/src/constants/index.js | 9 +++++++++ .../[selectedDatasource]/index.svelte | 4 ++++ packages/client/src/api/datasources.js | 3 ++- packages/client/src/api/rows.js | 3 ++- .../client/src/components/app/Layout.svelte | 9 +++++++-- .../app/forms/RelationshipField.svelte | 3 ++- .../src/components/app/forms/validation.js | 15 +++++++------- .../src/components/app/table/Table.svelte | 5 +++++ packages/client/src/constants.js | 20 +++++++++++++++++++ packages/client/src/stores/dataSource.js | 5 +++-- .../src/integrations/microsoftSqlServer.ts | 4 ++-- .../worker/src/api/controllers/global/auth.js | 3 +-- 14 files changed, 79 insertions(+), 20 deletions(-) diff --git a/packages/builder/src/components/backend/DataTable/Table.svelte b/packages/builder/src/components/backend/DataTable/Table.svelte index 512324254c..1ffac83b1a 100644 --- a/packages/builder/src/components/backend/DataTable/Table.svelte +++ b/packages/builder/src/components/backend/DataTable/Table.svelte @@ -8,7 +8,11 @@ import CreateEditRow from "./modals/CreateEditRow.svelte" import CreateEditUser from "./modals/CreateEditUser.svelte" import CreateEditColumn from "./modals/CreateEditColumn.svelte" - import { TableNames, UNEDITABLE_USER_FIELDS } from "constants" + import { + TableNames, + UNEDITABLE_USER_FIELDS, + UNSORTABLE_TYPES, + } from "constants" import RoleCell from "./cells/RoleCell.svelte" export let schema = {} @@ -33,6 +37,15 @@ $: isUsersTable = tableId === TableNames.USERS $: data && resetSelectedRows() $: editRowComponent = isUsersTable ? CreateEditUser : CreateEditRow + $: { + UNSORTABLE_TYPES.forEach(type => { + Object.values(schema).forEach(col => { + if (col.type === type) { + col.sortable = false + } + }) + }) + } $: { if (isUsersTable) { customRenderers = [ diff --git a/packages/builder/src/components/backend/DataTable/modals/CreateEditColumn.svelte b/packages/builder/src/components/backend/DataTable/modals/CreateEditColumn.svelte index 6ee0c48d2a..aa21799ca2 100644 --- a/packages/builder/src/components/backend/DataTable/modals/CreateEditColumn.svelte +++ b/packages/builder/src/components/backend/DataTable/modals/CreateEditColumn.svelte @@ -92,7 +92,6 @@ opt.type === table.type && table.sourceId === opt.sourceId ) - $: console.log(tableOptions) async function saveColumn() { if (field.type === AUTO_TYPE) { diff --git a/packages/builder/src/constants/index.js b/packages/builder/src/constants/index.js index c0d283b0ea..f13d2b80f0 100644 --- a/packages/builder/src/constants/index.js +++ b/packages/builder/src/constants/index.js @@ -1,3 +1,5 @@ +import { FIELDS } from "constants/backend" + export const TableNames = { USERS: "ta_users", } @@ -39,6 +41,13 @@ export const UNEDITABLE_USER_FIELDS = [ "lastName", ] +export const UNSORTABLE_TYPES = [ + FIELDS.FORMULA.type, + FIELDS.ATTACHMENT.type, + FIELDS.ARRAY.type, + FIELDS.LINK.type, +] + export const LAYOUT_NAMES = { MASTER: { PRIVATE: "layout_private_master", diff --git a/packages/builder/src/pages/builder/app/[application]/data/datasource/[selectedDatasource]/index.svelte b/packages/builder/src/pages/builder/app/[application]/data/datasource/[selectedDatasource]/index.svelte index b1867db248..6cbe0c8359 100644 --- a/packages/builder/src/pages/builder/app/[application]/data/datasource/[selectedDatasource]/index.svelte +++ b/packages/builder/src/pages/builder/app/[application]/data/datasource/[selectedDatasource]/index.svelte @@ -79,6 +79,10 @@ try { // Create datasource await datasources.save(datasource) + if (datasource?.plus) { + await tables.fetch() + } + await datasources.fetch() notifications.success(`Datasource ${name} updated successfully.`) } catch (err) { notifications.error(`Error saving datasource: ${err}`) diff --git a/packages/client/src/api/datasources.js b/packages/client/src/api/datasources.js index 508e1e8db0..05d06864e2 100644 --- a/packages/client/src/api/datasources.js +++ b/packages/client/src/api/datasources.js @@ -3,6 +3,7 @@ import { fetchTableData } from "./tables" import { fetchViewData } from "./views" import { fetchRelationshipData } from "./relationships" import { executeQuery } from "./queries" +import { FieldTypes } from "../constants" /** * Fetches all rows for a particular Budibase data source. @@ -28,7 +29,7 @@ export const fetchDatasource = async dataSource => { } } rows = await executeQuery({ queryId: dataSource._id, parameters }) - } else if (type === "link") { + } else if (type === FieldTypes.LINK) { rows = await fetchRelationshipData({ rowId: dataSource.rowId, tableId: dataSource.rowTableId, diff --git a/packages/client/src/api/rows.js b/packages/client/src/api/rows.js index acd083454d..7deec349e6 100644 --- a/packages/client/src/api/rows.js +++ b/packages/client/src/api/rows.js @@ -1,6 +1,7 @@ import { notificationStore, dataSourceStore } from "stores" import API from "./api" import { fetchTableDefinition } from "./tables" +import { FieldTypes } from "../constants" /** * Fetches data about a certain row in a table. @@ -129,7 +130,7 @@ export const enrichRows = async (rows, tableId) => { const keys = Object.keys(schema) for (let key of keys) { const type = schema[key].type - if (type === "link" && Array.isArray(row[key])) { + if (type === FieldTypes.LINK && Array.isArray(row[key])) { // Enrich row a string join of relationship fields row[`${key}_text`] = row[key] diff --git a/packages/client/src/components/app/Layout.svelte b/packages/client/src/components/app/Layout.svelte index 0ffbbf7ab1..87e5ac3b5b 100644 --- a/packages/client/src/components/app/Layout.svelte +++ b/packages/client/src/components/app/Layout.svelte @@ -1,6 +1,7 @@