From e6f3a04b4f532749a0f0517cd23f3b7e611576a2 Mon Sep 17 00:00:00 2001 From: Martin McKeaveney Date: Wed, 23 Jun 2021 14:29:40 +0100 Subject: [PATCH 01/41] started on opinionated relationships --- .../CreateEditRelationship.svelte | 79 +++++++++++++++++++ .../[selectedDatasource]/index.svelte | 61 +++++++++++--- 2 files changed, 127 insertions(+), 13 deletions(-) create mode 100644 packages/builder/src/pages/builder/app/[application]/data/datasource/[selectedDatasource]/CreateEditRelationship.svelte diff --git a/packages/builder/src/pages/builder/app/[application]/data/datasource/[selectedDatasource]/CreateEditRelationship.svelte b/packages/builder/src/pages/builder/app/[application]/data/datasource/[selectedDatasource]/CreateEditRelationship.svelte new file mode 100644 index 0000000000..b9f6dbf092 --- /dev/null +++ b/packages/builder/src/pages/builder/app/[application]/data/datasource/[selectedDatasource]/CreateEditRelationship.svelte @@ -0,0 +1,79 @@ + + + + {#if step === 0} + Select your table + {:else if step === 1} + + Step 2 + {:else if step === 2} + Step 3 + {/if} + + + \ No newline at end of file diff --git a/packages/builder/src/pages/builder/app/[application]/data/datasource/[selectedDatasource]/index.svelte b/packages/builder/src/pages/builder/app/[application]/data/datasource/[selectedDatasource]/index.svelte index f805a5724f..b5a084120e 100644 --- a/packages/builder/src/pages/builder/app/[application]/data/datasource/[selectedDatasource]/index.svelte +++ b/packages/builder/src/pages/builder/app/[application]/data/datasource/[selectedDatasource]/index.svelte @@ -1,16 +1,19 @@ + + + + {#if datasource && integration}
@@ -100,19 +111,43 @@ having to write any queries at all.
- {#if datasource.entities} - {#each Object.keys(datasource.entities) as entity} -
onClickTable(datasource.entities[entity])} - > -

{entity}

-

Primary Key: {datasource.entities[entity].primary}

-

→

-
- {/each} - {/if} + {#each plusTables as table} +
onClickTable(table)} + > +

{table.name}

+

Primary Key: {table.primary}

+

→

+
+ {/each}
+ + +
+ Relationships + +
+ + Tell budibase how your tables are related to get even more smart features. + +
+ {#each plusTables as table} + {#each Object.keys(table) as column} + {#if table[column].type === "link"} +
onClickTable(table[column])} + > +

{table[column].name}

+

Primary Key: {table[column].primary}

+

→

+
+ {/if} + {/each} + {/each} +
+ {/if}
From fd4403037d40f655294f932c08de4acfe95608e3 Mon Sep 17 00:00:00 2001 From: mike12345567 Date: Wed, 23 Jun 2021 19:05:32 +0100 Subject: [PATCH 02/41] WIP - basic override of foreign keys. --- .../src/api/controllers/row/external.js | 85 ++++++++++++++++--- .../server/src/api/controllers/table/utils.js | 11 ++- packages/server/src/integrations/base/sql.js | 26 +++++- packages/server/src/integrations/postgres.js | 14 +++ 4 files changed, 117 insertions(+), 19 deletions(-) diff --git a/packages/server/src/api/controllers/row/external.js b/packages/server/src/api/controllers/row/external.js index 896f5a78e2..79a347750f 100644 --- a/packages/server/src/api/controllers/row/external.js +++ b/packages/server/src/api/controllers/row/external.js @@ -1,6 +1,6 @@ const { makeExternalQuery } = require("./utils") -const { DataSourceOperation, SortDirection } = require("../../../constants") -const { getExternalTable } = require("../table/utils") +const { DataSourceOperation, SortDirection, FieldTypes, RelationshipTypes } = require("../../../constants") +const { getAllExternalTables } = require("../table/utils") const { breakExternalTableId, generateRowIdField, @@ -35,17 +35,56 @@ function generateIdForRow(row, table) { return generateRowIdField(idParts) } -function outputProcessing(rows, table) { +function updateRelationshipColumns(rows, row, relationships, allTables) { + const columns = {} + for (let relationship of relationships) { + const linkedTable = allTables[relationship.tableName] + if (!linkedTable) { + continue + } + const display = linkedTable.primaryDisplay + const related = {} + if (display && row[display]) { + related.primaryDisplay = row[display] + } + related._id = row[relationship.to] + columns[relationship.from] = related + } + for (let [column, related] of Object.entries(columns)) { + if (!Array.isArray(rows[row._id][column])) { + rows[row._id][column] = [] + } + rows[row._id][column].push(related) + } + return rows +} + +function outputProcessing(rows, table, relationships, allTables) { // if no rows this is what is returned? Might be PG only if (rows[0].read === true) { return [] } + let finalRows = {} for (let row of rows) { row._id = generateIdForRow(row, table) - row.tableId = table._id - row._rev = "rev" + // this is a relationship of some sort + if (finalRows[row._id]) { + finalRows = updateRelationshipColumns(finalRows, row, relationships, allTables) + continue + } + const thisRow = {} + // filter the row down to what is actually the row (not joined) + for (let fieldName of Object.keys(table.schema)) { + thisRow[fieldName] = row[fieldName] + } + thisRow._id = row._id + thisRow.tableId = table._id + thisRow._rev = "rev" + finalRows[thisRow._id] = thisRow + // do this at end once its been added to the final rows + finalRows = updateRelationshipColumns(finalRows, row, relationships, allTables) } - return rows + return Object.values(finalRows) } function buildFilters(id, filters, table) { @@ -83,6 +122,26 @@ function buildFilters(id, filters, table) { } } +function buildRelationships(table) { + const relationships = [] + for (let [fieldName, field] of Object.entries(table.schema)) { + if (field.type !== FieldTypes.LINK) { + continue + } + // TODO: through field + if (field.relationshipType === RelationshipTypes.MANY_TO_MANY) { + continue + } + const broken = breakExternalTableId(field.tableId) + relationships.push({ + from: fieldName, + to: field.fieldName, + tableName: broken.tableName, + }) + } + return relationships +} + async function handleRequest( appId, operation, @@ -90,12 +149,14 @@ async function handleRequest( { id, row, filters, sort, paginate } = {} ) { let { datasourceId, tableName } = breakExternalTableId(tableId) - const table = await getExternalTable(appId, datasourceId, tableName) + const tables = await getAllExternalTables(appId, datasourceId) + const table = tables[tableName] if (!table) { throw `Unable to process query, table "${tableName}" not defined.` } // clean up row on ingress using schema filters = buildFilters(id, filters, table) + const relationships = buildRelationships(table) row = inputProcessing(row, table) if ( operation === DataSourceOperation.DELETE && @@ -116,6 +177,7 @@ async function handleRequest( filters, sort, paginate, + relationships, body: row, // pass an id filter into extra, purely for mysql/returning extra: { @@ -126,9 +188,9 @@ async function handleRequest( const response = await makeExternalQuery(appId, json) // we searched for rows in someway if (operation === DataSourceOperation.READ && Array.isArray(response)) { - return outputProcessing(response, table) + return outputProcessing(response, table, relationships, tables) } else { - row = outputProcessing(response, table)[0] + row = outputProcessing(response, table, relationships, tables)[0] return { row, table } } } @@ -270,7 +332,4 @@ exports.validate = async () => { return { valid: true } } -exports.fetchEnrichedRow = async () => { - // TODO: How does this work - throw "Not Implemented" -} +exports.fetchEnrichedRow = async () => {} diff --git a/packages/server/src/api/controllers/table/utils.js b/packages/server/src/api/controllers/table/utils.js index cdfd390027..78dae60ab1 100644 --- a/packages/server/src/api/controllers/table/utils.js +++ b/packages/server/src/api/controllers/table/utils.js @@ -204,15 +204,18 @@ class TableSaveFunctions { } } -exports.getExternalTable = async (appId, datasourceId, tableName) => { +exports.getAllExternalTables = async (appId, datasourceId) => { const db = new CouchDB(appId) const datasource = await db.get(datasourceId) if (!datasource || !datasource.entities) { throw "Datasource is not configured fully." } - return Object.values(datasource.entities).find( - entity => entity.name === tableName - ) + return datasource.entities +} + +exports.getExternalTable = async (appId, datasourceId, tableName) => { + const entities = await exports.getAllExternalTables(appId, datasourceId) + return entities[tableName] } exports.TableSaveFunctions = TableSaveFunctions diff --git a/packages/server/src/integrations/base/sql.js b/packages/server/src/integrations/base/sql.js index c7cc95fc3e..08eac09f0b 100644 --- a/packages/server/src/integrations/base/sql.js +++ b/packages/server/src/integrations/base/sql.js @@ -55,6 +55,25 @@ function addFilters(query, filters) { return query } +function addRelationships(query, fromTable, relationships) { + if (!relationships) { + return query + } + for (let relationship of relationships) { + const from = `${fromTable}.${relationship.from}` + const to = `${relationship.tableName}.${relationship.to}` + if (!relationship.through) { + query = query.innerJoin(relationship.tableName, from, to) + } else { + const through = relationship + query = query + .innerJoin(through.tableName, from, through.from) + .innerJoin(relationship.tableName, to, through.to) + } + } + return query +} + function buildCreate(knex, json, opts) { const { endpoint, body } = json let query = knex(endpoint.entityId) @@ -67,8 +86,9 @@ function buildCreate(knex, json, opts) { } function buildRead(knex, json, limit) { - let { endpoint, resource, filters, sort, paginate } = json - let query = knex(endpoint.entityId) + let { endpoint, resource, filters, sort, paginate, relationships } = json + const tableName = endpoint.entityId + let query = knex(tableName) // select all if not specified if (!resource) { resource = { fields: [] } @@ -81,6 +101,8 @@ function buildRead(knex, json, limit) { } // handle where query = addFilters(query, filters) + // handle join + query = addRelationships(query, tableName, relationships) // handle sorting if (sort) { for (let [key, value] of Object.entries(sort)) { diff --git a/packages/server/src/integrations/postgres.js b/packages/server/src/integrations/postgres.js index 6266e6ca64..f2d1f08162 100644 --- a/packages/server/src/integrations/postgres.js +++ b/packages/server/src/integrations/postgres.js @@ -153,6 +153,20 @@ class PostgresIntegration extends Sql { name: columnName, type: convertType(column.data_type, TYPE_MAP), } + + // // TODO: hack for testing + // if (tableName === "persons") { + // tables[tableName].primaryDisplay = "firstname" + // } + // if (columnName.toLowerCase() === "personid" && tableName === "tasks") { + // tables[tableName].schema[columnName] = { + // name: columnName, + // type: "link", + // tableId: buildExternalTableId(datasourceId, "persons"), + // relationshipType: "one-to-many", + // fieldName: "personid", + // } + // } } this.tables = tables } From 751f83bfb071dbbde9cc2d5ad83809b7064cb782 Mon Sep 17 00:00:00 2001 From: Martin McKeaveney Date: Wed, 23 Jun 2021 22:07:18 +0100 Subject: [PATCH 03/41] adding relationships file --- .../CreateEditRelationship.svelte | 156 ++++++++++++++++++ .../CreateEditRelationship/TableSelect.svelte | 21 +++ 2 files changed, 177 insertions(+) create mode 100644 packages/builder/src/pages/builder/app/[application]/data/datasource/[selectedDatasource]/CreateEditRelationship/CreateEditRelationship.svelte create mode 100644 packages/builder/src/pages/builder/app/[application]/data/datasource/[selectedDatasource]/CreateEditRelationship/TableSelect.svelte diff --git a/packages/builder/src/pages/builder/app/[application]/data/datasource/[selectedDatasource]/CreateEditRelationship/CreateEditRelationship.svelte b/packages/builder/src/pages/builder/app/[application]/data/datasource/[selectedDatasource]/CreateEditRelationship/CreateEditRelationship.svelte new file mode 100644 index 0000000000..cbab467e40 --- /dev/null +++ b/packages/builder/src/pages/builder/app/[application]/data/datasource/[selectedDatasource]/CreateEditRelationship/CreateEditRelationship.svelte @@ -0,0 +1,156 @@ + + + + + +
+ + + {#each tables as table} + (relationship.from = table)}> + {table.name} + {#if relationship.from?._id === table._id} + + {/if} + + {/each} + + + + + {#each tables as table} + (relationship.to = table)}> + {table.name} + {#if relationship.to?._id === table._id} + + {/if} + + {/each} + + +
+ + {#if relationship.from && relationship.to} +
+ option.name} + getOptionValue={option => option.value} + /> +
+ {/if} + + {#if relationship?.relationshipType === RelationshipTypes.MANY_TO_MANY} + + + {#each tables as table} + (relationship.through = table)}> + {table.name} + {#if relationship.through?._id === table._id} + + {/if} + + {/each} + + + + {#if relationship.through} +
+ + + {#each Object.keys(relationship.through) as column} + (relationship.through.from = column)}> + {column} + {#if relationship.through.from?._id === column._id} + + {/if} + + {/each} + + + + + {#each Object.keys(relationship.through) as column} + (relationship.through.to = column)}> + {column} + {#if relationship.through.to?._id === column._id} + + {/if} + + {/each} + + +
+ {/if} + {/if} + + + \ No newline at end of file diff --git a/packages/builder/src/pages/builder/app/[application]/data/datasource/[selectedDatasource]/CreateEditRelationship/TableSelect.svelte b/packages/builder/src/pages/builder/app/[application]/data/datasource/[selectedDatasource]/CreateEditRelationship/TableSelect.svelte new file mode 100644 index 0000000000..8d4613e0b5 --- /dev/null +++ b/packages/builder/src/pages/builder/app/[application]/data/datasource/[selectedDatasource]/CreateEditRelationship/TableSelect.svelte @@ -0,0 +1,21 @@ + + + + + {#each tables as table} + select(table)}> + {table.name} + {#if selected} + + {/if} + + {/each} + + \ No newline at end of file From aa980e9b8aa31001ae4d8333c7381196204e331f Mon Sep 17 00:00:00 2001 From: Martin McKeaveney Date: Fri, 25 Jun 2021 18:11:03 +0100 Subject: [PATCH 04/41] relationship configuration panel --- .../CreateEditRelationship.svelte | 79 ------------- .../CreateEditRelationship.svelte | 104 +++++++++--------- .../[selectedDatasource]/index.svelte | 2 +- 3 files changed, 53 insertions(+), 132 deletions(-) delete mode 100644 packages/builder/src/pages/builder/app/[application]/data/datasource/[selectedDatasource]/CreateEditRelationship.svelte diff --git a/packages/builder/src/pages/builder/app/[application]/data/datasource/[selectedDatasource]/CreateEditRelationship.svelte b/packages/builder/src/pages/builder/app/[application]/data/datasource/[selectedDatasource]/CreateEditRelationship.svelte deleted file mode 100644 index b9f6dbf092..0000000000 --- a/packages/builder/src/pages/builder/app/[application]/data/datasource/[selectedDatasource]/CreateEditRelationship.svelte +++ /dev/null @@ -1,79 +0,0 @@ - - - - {#if step === 0} - Select your table - {:else if step === 1} - - Step 2 - {:else if step === 2} - Step 3 - {/if} - - - \ No newline at end of file diff --git a/packages/builder/src/pages/builder/app/[application]/data/datasource/[selectedDatasource]/CreateEditRelationship/CreateEditRelationship.svelte b/packages/builder/src/pages/builder/app/[application]/data/datasource/[selectedDatasource]/CreateEditRelationship/CreateEditRelationship.svelte index cbab467e40..0ed5a93c0a 100644 --- a/packages/builder/src/pages/builder/app/[application]/data/datasource/[selectedDatasource]/CreateEditRelationship/CreateEditRelationship.svelte +++ b/packages/builder/src/pages/builder/app/[application]/data/datasource/[selectedDatasource]/CreateEditRelationship/CreateEditRelationship.svelte @@ -2,33 +2,44 @@ import { RelationshipTypes } from "constants/backend" import { Menu, MenuItem, MenuSection, Button, Input, Icon, ModalContent, RadioGroup, Heading } from "@budibase/bbui" + // "tasks_something": { + // "name": "tasks_something", + // "type": "link", + // "tableId": "whatever/othertable", + // "relationshipType": "one-to-many", + // }, + export let save export let datasource + export let from export let tables - - let relationship = {} + export let relationship = {} $: console.log(relationship) - $: console.log("ds", datasource) - $: valid = relationship.name && relationship.from && relationship.to && relationship.relationshipType + $: valid = relationship.name && relationship.tableId && relationship.relationshipType + $: from = tables.find(table => table._id === relationship.source) + $: to = tables.find(table => table._id === relationship.tableId) + $: through = tables.find(table => table._id === relationship.through) + $: linkTable = through || to - $: relationshipOptions = relationship.from && relationship.to ? [ + $: relationshipOptions = from && to ? [ { - name: `Many ${relationship.from.name} rows → many ${relationship.to.name} rows`, - alt: `Many ${relationship.from.name} rows → many ${relationship.to.name} rows`, + name: `Many ${from.name} rows → many ${to.name} rows`, + alt: `Many ${from.name} rows → many ${to.name} rows`, value: RelationshipTypes.MANY_TO_MANY, }, { - name: `One ${relationship.from.name} row → many ${relationship.to.name} rows`, - alt: `One ${relationship.from.name} rows → many ${relationship.to.name} rows`, + name: `One ${from.name} row → many ${to.name} rows`, + alt: `One ${from.name} rows → many ${to.name} rows`, value: RelationshipTypes.ONE_TO_MANY, - }, - { - name: `One ${relationship.from.name} row → many ${relationship.to.name} rows`, - alt: `One ${relationship.from.name} row → many ${relationship.to.name} rows`, - value: RelationshipTypes.MANY_TO_ONE, - }, + } ] : [] + + function onChangeRelationshipType(evt) { + if (evt.detail === RelationshipTypes.ONE_TO_MANY) { + relationship.through = null + } + } // save the relationship on to the datasource function saveRelationship() { @@ -67,9 +78,9 @@ {#each tables as table} - (relationship.from = table)}> + (relationship.source = table._id)}> {table.name} - {#if relationship.from?._id === table._id} + {#if relationship.source === table._id} {/if} @@ -79,9 +90,9 @@ {#each tables as table} - (relationship.to = table)}> + (relationship.tableId = table._id)}> {table.name} - {#if relationship.to?._id === table._id} + {#if relationship.tableId === table._id} {/if} @@ -90,61 +101,50 @@
- {#if relationship.from && relationship.to} + {#if from && to}
option.name} getOptionValue={option => option.value} />
- {/if} {#if relationship?.relationshipType === RelationshipTypes.MANY_TO_MANY} {#each tables as table} - (relationship.through = table)}> + (relationship.through = table._id)}> {table.name} - {#if relationship.through?._id === table._id} + {#if relationship.through === table._id} {/if} {/each} - - {#if relationship.through} -
- - - {#each Object.keys(relationship.through) as column} - (relationship.through.from = column)}> - {column} - {#if relationship.through.from?._id === column._id} - - {/if} - - {/each} - - - - - {#each Object.keys(relationship.through) as column} - (relationship.through.to = column)}> - {column} - {#if relationship.through.to?._id === column._id} - - {/if} - - {/each} - - -
- {/if} {/if} + + +
+ + + {#each Object.keys(linkTable.schema) as column} + (relationship.foreignKey = column)}> + {column} + {#if relationship.foreignKey === column} + + {/if} + + {/each} + + +
+ {/if} + \ No newline at end of file diff --git a/packages/server/.vscode/launch.json b/packages/server/.vscode/launch.json index 7417938376..d1774752cb 100644 --- a/packages/server/.vscode/launch.json +++ b/packages/server/.vscode/launch.json @@ -8,7 +8,19 @@ "type": "node", "request": "launch", "name": "Start Server", - "program": "${workspaceFolder}/src/index.js" + "sourceMaps": true, + "program": "${workspaceFolder}/src/index.ts", + "preLaunchTask": "npm: build", + "outFiles": ["${workspaceFolder}/dist/**/*.js"] + }, + { + "name": "TS", + "type": "node", + "request": "launch", + "runtimeExecutable": "node", + "runtimeArgs": ["--nolazy", "-r", "ts-node/register/transpile-only"], + "args": ["src/index.ts"], + "cwd": "${workspaceRoot}", }, { "type": "node", From 7fb1e8019605ffaf62cece971346f0f4b95049c9 Mon Sep 17 00:00:00 2001 From: mike12345567 Date: Thu, 1 Jul 2021 19:20:41 +0100 Subject: [PATCH 25/41] Re-writing external layer in typescript. --- .../api/controllers/row/ExternalRequest.ts | 424 ++++++++++++++++++ .../src/api/controllers/row/external.js | 108 +---- .../src/api/controllers/row/externalUtils.js | 259 ----------- packages/server/src/definitions/common.ts | 42 +- packages/server/src/definitions/datasource.ts | 25 +- packages/server/src/integrations/utils.ts | 3 - 6 files changed, 463 insertions(+), 398 deletions(-) create mode 100644 packages/server/src/api/controllers/row/ExternalRequest.ts delete mode 100644 packages/server/src/api/controllers/row/externalUtils.js diff --git a/packages/server/src/api/controllers/row/ExternalRequest.ts b/packages/server/src/api/controllers/row/ExternalRequest.ts new file mode 100644 index 0000000000..950cbef084 --- /dev/null +++ b/packages/server/src/api/controllers/row/ExternalRequest.ts @@ -0,0 +1,424 @@ +import { + Operation, + SearchFilters, + SortJson, + PaginationJson, + RelationshipsJson, +} from "../../../definitions/datasource" +import { Row, Table, FieldSchema } from "../../../definitions/common" +import { + breakRowIdField, + generateRowIdField, +} from "../../../integrations/utils" + +interface ManyRelationship { + tableId?: string + id?: string + isUpdate?: boolean + [key: string]: any +} + +interface RunConfig { + id: string + row: Row + filters: SearchFilters + sort: SortJson + paginate: PaginationJson +} + +module External { + const { makeExternalQuery } = require("./utils") + const { DataSourceOperation, FieldTypes } = require("../../../constants") + const { getAllExternalTables } = require("../table/utils") + const { breakExternalTableId } = require("../../../integrations/utils") + const { processObjectSync } = require("@budibase/string-templates") + const { cloneDeep } = require("lodash/fp") + + function buildFilters( + id: string | undefined, + filters: SearchFilters, + table: Table + ) { + const primary = table.primary + // if passed in array need to copy for shifting etc + let idCopy = cloneDeep(id) + if (filters) { + // need to map over the filters and make sure the _id field isn't present + for (let filter of Object.values(filters)) { + if (filter._id && primary) { + const parts = breakRowIdField(filter._id) + for (let field of primary) { + filter[field] = parts.shift() + } + } + // make sure this field doesn't exist on any filter + delete filter._id + } + } + // there is no id, just use the user provided filters + if (!idCopy || !table) { + return filters + } + // if used as URL parameter it will have been joined + if (!Array.isArray(idCopy)) { + idCopy = breakRowIdField(idCopy) + } + const equal: any = {} + if (primary && idCopy) { + for (let field of primary) { + // work through the ID and get the parts + equal[field] = idCopy.shift() + } + } + return { + equal, + } + } + + function generateIdForRow(row: Row, table: Table): string { + const primary = table.primary + if (!row || !primary) { + return "" + } + // build id array + let idParts = [] + for (let field of primary) { + if (row[field]) { + idParts.push(row[field]) + } + } + if (idParts.length === 0) { + return "" + } + return generateRowIdField(idParts) + } + + function basicProcessing(row: Row, table: Table) { + const thisRow: { [key: string]: any } = {} + // filter the row down to what is actually the row (not joined) + for (let fieldName of Object.keys(table.schema)) { + thisRow[fieldName] = row[fieldName] + } + thisRow._id = generateIdForRow(row, table) + thisRow.tableId = table._id + thisRow._rev = "rev" + return thisRow + } + + function isMany(field: FieldSchema) { + return ( + field.relationshipType && field.relationshipType.split("-")[0] === "many" + ) + } + + class ExternalRequest { + private appId: string + private operation: Operation + private tableId: string + private tables: { [key: string]: Table } + + constructor( + appId: string, + operation: Operation, + tableId: string, + tables: { [key: string]: Table } + ) { + this.appId = appId + this.operation = operation + this.tableId = tableId + this.tables = tables + } + + inputProcessing(row: Row, table: Table) { + if (!row) { + return { row, manyRelationships: [] } + } + // we don't really support composite keys for relationships, this is why [0] is used + // @ts-ignore + const tablePrimary: string = table.primary[0] + let newRow: Row = {}, + manyRelationships: ManyRelationship[] = [] + for (let [key, field] of Object.entries(table.schema)) { + // if set already, or not set just skip it + if (!row[key] || newRow[key]) { + continue + } + // if its not a link then just copy it over + if (field.type !== FieldTypes.LINK) { + newRow[key] = row[key] + continue + } + const { tableName: linkTableName } = breakExternalTableId(field.tableId) + // table has to exist for many to many + if (!this.tables[linkTableName]) { + continue + } + const linkTable = this.tables[linkTableName] + // @ts-ignore + const linkTablePrimary = linkTable.primary[0] + if (!isMany(field)) { + + newRow[field.foreignKey || linkTablePrimary] = breakRowIdField( + row[key][0] + )[0] + } else { + // we're not inserting a doc, will be a bunch of update calls + const isUpdate = !field.through + const thisKey: string = isUpdate ? "id" : linkTablePrimary + // @ts-ignore + const otherKey: string = isUpdate + ? field.foreignKey + : tablePrimary + row[key].map((relationship: any) => { + // we don't really support composite keys for relationships, this is why [0] is used + manyRelationships.push({ + tableId: field.through || field.tableId, + isUpdate, + [thisKey]: breakRowIdField(relationship)[0], + // leave the ID for enrichment later + [otherKey]: `{{ ${tablePrimary} }}`, + }) + }) + } + } + // we return the relationships that may need to be created in the through table + // we do this so that if the ID is generated by the DB it can be inserted + // after the fact + return { row: newRow, manyRelationships } + } + + updateRelationshipColumns( + row: Row, + rows: { [key: string]: Row }, + relationships: RelationshipsJson[] + ) { + const columns: { [key: string]: any } = {} + for (let relationship of relationships) { + const linkedTable = this.tables[relationship.tableName] + if (!linkedTable) { + continue + } + let linked = basicProcessing(row, linkedTable) + if (!linked._id) { + continue + } + // if not returning full docs then get the minimal links out + const display = linkedTable.primaryDisplay + linked = { + primaryDisplay: display ? linked[display] : undefined, + _id: linked._id, + } + columns[relationship.column] = linked + } + for (let [column, related] of Object.entries(columns)) { + if (!row._id) { + continue + } + const rowId: string = row._id + if (!Array.isArray(rows[rowId][column])) { + rows[rowId][column] = [] + } + // make sure relationship hasn't been found already + if ( + !rows[rowId][column].find( + (relation: Row) => relation._id === related._id + ) + ) { + rows[rowId][column].push(related) + } + } + return rows + } + + outputProcessing( + rows: Row[], + table: Table, + relationships: RelationshipsJson[] + ) { + if (rows[0].read === true) { + return [] + } + let finalRows: { [key: string]: Row } = {} + for (let row of rows) { + const rowId = generateIdForRow(row, table) + row._id = rowId + // this is a relationship of some sort + if (finalRows[rowId]) { + finalRows = this.updateRelationshipColumns( + row, + finalRows, + relationships + ) + continue + } + const thisRow = basicProcessing(row, table) + finalRows[thisRow._id] = thisRow + // do this at end once its been added to the final rows + finalRows = this.updateRelationshipColumns( + row, + finalRows, + relationships + ) + } + return Object.values(finalRows) + } + + buildRelationships(table: Table): RelationshipsJson[] { + const relationships = [] + for (let [fieldName, field] of Object.entries(table.schema)) { + if (field.type !== FieldTypes.LINK) { + continue + } + const { tableName: linkTableName } = breakExternalTableId(field.tableId) + // no table to link to, this is not a valid relationships + if (!this.tables[linkTableName]) { + continue + } + const linkTable = this.tables[linkTableName] + if (!table.primary || !linkTable.primary) { + continue + } + const definition = { + // if no foreign key specified then use the name of the field in other table + from: field.foreignKey || table.primary[0], + to: field.fieldName, + tableName: linkTableName, + through: undefined, + // need to specify where to put this back into + column: fieldName, + } + if (field.through) { + const { tableName: throughTableName } = breakExternalTableId( + field.through + ) + definition.through = throughTableName + // don't support composite keys for relationships + definition.from = table.primary[0] + definition.to = linkTable.primary[0] + } + relationships.push(definition) + } + return relationships + } + + async handleManyRelationships(row: Row, relationships: ManyRelationship[]) { + const { appId, tables } = this + const promises = [] + for (let relationship of relationships) { + const { tableId, isUpdate, id, ...rest } = relationship + const { datasourceId, tableName } = breakExternalTableId(tableId) + const linkedTable = tables[tableName] + if (!linkedTable) { + continue + } + const endpoint = { + datasourceId, + entityId: tableName, + operation: isUpdate + ? DataSourceOperation.UPDATE + : DataSourceOperation.CREATE, + } + promises.push( + makeExternalQuery(appId, { + endpoint, + // if we're doing many relationships then we're writing, only one response + body: processObjectSync(rest, row), + filters: buildFilters(id, {}, linkedTable), + }) + ) + } + await Promise.all(promises) + } + + /** + * This function is a bit crazy, but the exact purpose of it is to protect against the scenario in which + * you have column overlap in relationships, e.g. we join a few different tables and they all have the + * concept of an ID, but for some of them it will be null (if they say don't have a relationship). + * Creating the specific list of fields that we desire, and excluding the ones that are no use to us + * is more performant and has the added benefit of protecting against this scenario. + */ + buildFields(table: Table) { + function extractNonLinkFieldNames(table: Table, existing: string[] = []) { + return Object.entries(table.schema) + .filter( + column => + column[1].type !== FieldTypes.LINK && + !existing.find((field: string) => field.includes(column[0])) + ) + .map(column => `${table.name}.${column[0]}`) + } + let fields = extractNonLinkFieldNames(table) + for (let field of Object.values(table.schema)) { + if (field.type !== FieldTypes.LINK) { + continue + } + const { tableName: linkTableName } = breakExternalTableId(field.tableId) + const linkTable = this.tables[linkTableName] + if (linkTable) { + const linkedFields = extractNonLinkFieldNames(linkTable, fields) + fields = fields.concat(linkedFields) + } + } + return fields + } + + async run({ id, row, filters, sort, paginate }: RunConfig) { + const { appId, operation, tableId } = this + let { datasourceId, tableName } = breakExternalTableId(tableId) + if (!this.tables) { + this.tables = await getAllExternalTables(appId, datasourceId) + } + const table = this.tables[tableName] + if (!table) { + throw `Unable to process query, table "${tableName}" not defined.` + } + // clean up row on ingress using schema + filters = buildFilters(id, filters, table) + const relationships = this.buildRelationships(table) + const processed = this.inputProcessing(row, table) + row = processed.row + if ( + operation === DataSourceOperation.DELETE && + (filters == null || Object.keys(filters).length === 0) + ) { + throw "Deletion must be filtered" + } + let json = { + endpoint: { + datasourceId, + entityId: tableName, + operation, + }, + resource: { + // have to specify the fields to avoid column overlap + fields: this.buildFields(table), + }, + filters, + sort, + paginate, + relationships, + body: row, + // pass an id filter into extra, purely for mysql/returning + extra: { + idFilter: buildFilters(id || generateIdForRow(row, table), {}, table), + }, + } + // can't really use response right now + const response = await makeExternalQuery(appId, json) + // handle many to many relationships now if we know the ID (could be auto increment) + if (processed.manyRelationships) { + await this.handleManyRelationships( + response[0], + processed.manyRelationships + ) + } + const output = this.outputProcessing(response, table, relationships) + // if reading it'll just be an array of rows, return whole thing + return operation === DataSourceOperation.READ && Array.isArray(response) + ? output + : { row: output[0], table } + } + } + + module.exports = ExternalRequest +} diff --git a/packages/server/src/api/controllers/row/external.js b/packages/server/src/api/controllers/row/external.js index 7786dbcd45..96928fb2ae 100644 --- a/packages/server/src/api/controllers/row/external.js +++ b/packages/server/src/api/controllers/row/external.js @@ -1,4 +1,3 @@ -const { makeExternalQuery } = require("./utils") const { DataSourceOperation, SortDirection, @@ -9,112 +8,9 @@ const { breakExternalTableId, breakRowIdField, } = require("../../../integrations/utils") -const { - buildRelationships, - buildFilters, - inputProcessing, - outputProcessing, - generateIdForRow, - buildFields, -} = require("./externalUtils") -const { processObjectSync } = require("@budibase/string-templates") +const ExternalRequest = require("./ExternalRequest") -class ExternalRequest { - constructor(appId, operation, tableId, tables) { - this.appId = appId - this.operation = operation - this.tableId = tableId - this.tables = tables - } - - async handleManyRelationships(row, relationships) { - const { appId, tables } = this - const promises = [] - for (let relationship of relationships) { - const { tableId, isUpdate, id, ...rest } = relationship - const { datasourceId, tableName } = breakExternalTableId(tableId) - const linkedTable = tables[tableName] - if (!linkedTable) { - continue - } - const endpoint = { - datasourceId, - entityId: tableName, - operation: isUpdate ? DataSourceOperation.UPDATE : DataSourceOperation.CREATE, - } - promises.push( - makeExternalQuery(appId, { - endpoint, - // if we're doing many relationships then we're writing, only one response - body: processObjectSync(rest, row), - filters: buildFilters(id, {}, linkedTable) - }) - ) - } - await Promise.all(promises) - } - - async run({ id, row, filters, sort, paginate }) { - const { appId, operation, tableId } = this - let { datasourceId, tableName } = breakExternalTableId(tableId) - if (!this.tables) { - this.tables = await getAllExternalTables(appId, datasourceId) - } - const table = this.tables[tableName] - if (!table) { - throw `Unable to process query, table "${tableName}" not defined.` - } - // clean up row on ingress using schema - filters = buildFilters(id, filters, table) - const relationships = buildRelationships(table, this.tables) - const processed = inputProcessing(row, table, this.tables) - row = processed.row - if ( - operation === DataSourceOperation.DELETE && - (filters == null || Object.keys(filters).length === 0) - ) { - throw "Deletion must be filtered" - } - let json = { - endpoint: { - datasourceId, - entityId: tableName, - operation, - }, - resource: { - // have to specify the fields to avoid column overlap - fields: buildFields(table, this.tables), - }, - filters, - sort, - paginate, - relationships, - body: row, - // pass an id filter into extra, purely for mysql/returning - extra: { - idFilter: buildFilters(id || generateIdForRow(row, table), {}, table), - }, - } - // can't really use response right now - const response = await makeExternalQuery(appId, json) - // handle many to many relationships now if we know the ID (could be auto increment) - if (processed.manyRelationships) { - await this.handleManyRelationships(response[0], processed.manyRelationships) - } - const output = outputProcessing(response, table, relationships, this.tables) - // if reading it'll just be an array of rows, return whole thing - return operation === DataSourceOperation.READ && Array.isArray(response) - ? output - : { row: output[0], table } - } -} - -async function handleRequest( - appId, - operation, - tableId, - opts = {} -) { +async function handleRequest(appId, operation, tableId, opts = {}) { return new ExternalRequest(appId, operation, tableId, opts.tables).run(opts) } diff --git a/packages/server/src/api/controllers/row/externalUtils.js b/packages/server/src/api/controllers/row/externalUtils.js deleted file mode 100644 index 71f80ddb5a..0000000000 --- a/packages/server/src/api/controllers/row/externalUtils.js +++ /dev/null @@ -1,259 +0,0 @@ -const { - breakExternalTableId, - generateRowIdField, - breakRowIdField, -} = require("../../../integrations/utils") -const { FieldTypes } = require("../../../constants") -const { cloneDeep } = require("lodash/fp") - -function basicProcessing(row, table) { - const thisRow = {} - // filter the row down to what is actually the row (not joined) - for (let fieldName of Object.keys(table.schema)) { - thisRow[fieldName] = row[fieldName] - } - thisRow._id = exports.generateIdForRow(row, table) - thisRow.tableId = table._id - thisRow._rev = "rev" - return thisRow -} - -function isMany(field) { - return field.relationshipType.split("-")[0] === "many" -} - -exports.inputProcessing = (row, table, allTables) => { - if (!row) { - return { row, manyRelationships: [] } - } - let newRow = {}, - manyRelationships = [] - for (let [key, field] of Object.entries(table.schema)) { - // if set already, or not set just skip it - if (!row[key] || newRow[key]) { - continue - } - // if its not a link then just copy it over - if (field.type !== FieldTypes.LINK) { - newRow[key] = row[key] - continue - } - const { tableName: linkTableName } = breakExternalTableId(field.tableId) - // table has to exist for many to many - if (!allTables[linkTableName]) { - continue - } - const linkTable = allTables[linkTableName] - if (!isMany(field)) { - // we don't really support composite keys for relationships, this is why [0] is used - newRow[field.foreignKey || linkTable.primary] = breakRowIdField( - row[key][0] - )[0] - } else { - // we're not inserting a doc, will be a bunch of update calls - const isUpdate = !field.through - const thisKey = isUpdate ? "id" : linkTable.primary - const otherKey = isUpdate ? field.foreignKey : table.primary - row[key].map(relationship => { - // we don't really support composite keys for relationships, this is why [0] is used - manyRelationships.push({ - tableId: field.through || field.tableId, - isUpdate, - [thisKey]: breakRowIdField(relationship)[0], - // leave the ID for enrichment later - [otherKey]: `{{ ${table.primary} }}`, - }) - }) - } - } - // we return the relationships that may need to be created in the through table - // we do this so that if the ID is generated by the DB it can be inserted - // after the fact - return { row: newRow, manyRelationships } -} - -exports.generateIdForRow = (row, table) => { - if (!row) { - return null - } - const primary = table.primary - // build id array - let idParts = [] - for (let field of primary) { - if (row[field]) { - idParts.push(row[field]) - } - } - if (idParts.length === 0) { - return null - } - return generateRowIdField(idParts) -} - -exports.updateRelationshipColumns = (row, rows, relationships, allTables) => { - const columns = {} - for (let relationship of relationships) { - const linkedTable = allTables[relationship.tableName] - if (!linkedTable) { - continue - } - let linked = basicProcessing(row, linkedTable) - if (!linked._id) { - continue - } - // if not returning full docs then get the minimal links out - const display = linkedTable.primaryDisplay - linked = { - primaryDisplay: display ? linked[display] : undefined, - _id: linked._id, - } - columns[relationship.column] = linked - } - for (let [column, related] of Object.entries(columns)) { - if (!Array.isArray(rows[row._id][column])) { - rows[row._id][column] = [] - } - // make sure relationship hasn't been found already - if (!rows[row._id][column].find(relation => relation._id === related._id)) { - rows[row._id][column].push(related) - } - } - return rows -} - -exports.outputProcessing = (rows, table, relationships, allTables) => { - // if no rows this is what is returned? Might be PG only - if (rows[0].read === true) { - return [] - } - let finalRows = {} - for (let row of rows) { - row._id = exports.generateIdForRow(row, table) - // this is a relationship of some sort - if (finalRows[row._id]) { - finalRows = exports.updateRelationshipColumns( - row, - finalRows, - relationships, - allTables - ) - continue - } - const thisRow = basicProcessing(row, table) - finalRows[thisRow._id] = thisRow - // do this at end once its been added to the final rows - finalRows = exports.updateRelationshipColumns( - row, - finalRows, - relationships, - allTables - ) - } - return Object.values(finalRows) -} - -/** - * This function is a bit crazy, but the exact purpose of it is to protect against the scenario in which - * you have column overlap in relationships, e.g. we join a few different tables and they all have the - * concept of an ID, but for some of them it will be null (if they say don't have a relationship). - * Creating the specific list of fields that we desire, and excluding the ones that are no use to us - * is more performant and has the added benefit of protecting against this scenario. - * @param {Object} table The table we are retrieving fields for. - * @param {Object[]} allTables All of the tables that exist in the external data source, this is - * needed to work out what is needed from other tables based on relationships. - * @return {string[]} A list of fields like ["products.productid"] which can be used for an SQL select. - */ -exports.buildFields = (table, allTables) => { - function extractNonLinkFieldNames(table, existing = []) { - return Object.entries(table.schema) - .filter( - column => - column[1].type !== FieldTypes.LINK && - !existing.find(field => field.includes(column[0])) - ) - .map(column => `${table.name}.${column[0]}`) - } - let fields = extractNonLinkFieldNames(table) - for (let field of Object.values(table.schema)) { - if (field.type !== FieldTypes.LINK) { - continue - } - const { tableName: linkTableName } = breakExternalTableId(field.tableId) - const linkTable = allTables[linkTableName] - if (linkTable) { - const linkedFields = extractNonLinkFieldNames(linkTable, fields) - fields = fields.concat(linkedFields) - } - } - return fields -} - -exports.buildFilters = (id, filters, table) => { - const primary = table.primary - // if passed in array need to copy for shifting etc - let idCopy = cloneDeep(id) - if (filters) { - // need to map over the filters and make sure the _id field isn't present - for (let filter of Object.values(filters)) { - if (filter._id) { - const parts = breakRowIdField(filter._id) - for (let field of primary) { - filter[field] = parts.shift() - } - } - // make sure this field doesn't exist on any filter - delete filter._id - } - } - // there is no id, just use the user provided filters - if (!idCopy || !table) { - return filters - } - // if used as URL parameter it will have been joined - if (!Array.isArray(idCopy)) { - idCopy = breakRowIdField(idCopy) - } - const equal = {} - for (let field of primary) { - // work through the ID and get the parts - equal[field] = idCopy.shift() - } - return { - equal, - } -} - -exports.buildRelationships = (table, allTables) => { - const relationships = [] - for (let [fieldName, field] of Object.entries(table.schema)) { - if (field.type !== FieldTypes.LINK) { - continue - } - const { tableName: linkTableName } = breakExternalTableId(field.tableId) - // no table to link to, this is not a valid relationships - if (!allTables[linkTableName]) { - continue - } - const linkTable = allTables[linkTableName] - const definition = { - // if no foreign key specified then use the name of the field in other table - from: field.foreignKey || table.primary[0], - to: field.fieldName, - tableName: linkTableName, - through: undefined, - // need to specify where to put this back into - column: fieldName, - } - if (field.through) { - const { tableName: throughTableName } = breakExternalTableId( - field.through - ) - definition.through = throughTableName - // don't support composite keys for relationships - definition.from = table.primary[0] - definition.to = linkTable.primary[0] - } - relationships.push(definition) - } - return relationships -} diff --git a/packages/server/src/definitions/common.ts b/packages/server/src/definitions/common.ts index 26c12bd3c9..0115cc7685 100644 --- a/packages/server/src/definitions/common.ts +++ b/packages/server/src/definitions/common.ts @@ -3,29 +3,31 @@ interface Base { _rev?: string } -export interface TableSchema { - [key: string]: { - // TODO: replace with field types enum when done - type: string - fieldName?: string - name: string - tableId?: string - relationshipType?: string - through?: string - foreignKey?: string - constraints?: { - type?: string - email?: boolean - inclusion?: string[] - length?: { - minimum?: string | number - maximum?: string | number - } - presence?: boolean +export interface FieldSchema { + // TODO: replace with field types enum when done + type: string + fieldName?: string + name: string + tableId?: string + relationshipType?: string + through?: string + foreignKey?: string + constraints?: { + type?: string + email?: boolean + inclusion?: string[] + length?: { + minimum?: string | number + maximum?: string | number } + presence?: boolean } } +export interface TableSchema { + [key: string]: FieldSchema +} + export interface Table extends Base { type?: string views?: {} @@ -38,7 +40,7 @@ export interface Table extends Base { export interface Row extends Base { type?: string - tableId: string + tableId?: string [key: string]: any } diff --git a/packages/server/src/definitions/datasource.ts b/packages/server/src/definitions/datasource.ts index b7a3d47fff..ac52a32026 100644 --- a/packages/server/src/definitions/datasource.ts +++ b/packages/server/src/definitions/datasource.ts @@ -47,7 +47,7 @@ export interface Integration { } export interface SearchFilters { - allOr: boolean + allOr?: boolean string?: { [key: string]: string } @@ -77,11 +77,21 @@ export interface SearchFilters { } } +export interface SortJson { + [key: string]: SortDirection +} + +export interface PaginationJson { + limit: number + page: string | number +} + export interface RelationshipsJson { through?: string - from: string - to: string + from?: string + to?: string tableName: string + column: string } export interface QueryJson { @@ -94,13 +104,8 @@ export interface QueryJson { fields: string[] } filters?: SearchFilters - sort?: { - [key: string]: SortDirection - } - paginate?: { - limit: number - page: string | number - } + sort?: SortJson + paginate?: PaginationJson body?: object extra?: { idFilter?: SearchFilters diff --git a/packages/server/src/integrations/utils.ts b/packages/server/src/integrations/utils.ts index b668e6edeb..b8e9ee1dda 100644 --- a/packages/server/src/integrations/utils.ts +++ b/packages/server/src/integrations/utils.ts @@ -30,9 +30,6 @@ export function generateRowIdField(keyProps: any[] = []) { // should always return an array export function breakRowIdField(_id: string) { - if (!_id) { - return null - } const decoded = decodeURIComponent(_id) const parsed = JSON.parse(decoded) return Array.isArray(parsed) ? parsed : [parsed] From a5c3d4f86fc07628cc1caca4ea32e1971b315f69 Mon Sep 17 00:00:00 2001 From: mike12345567 Date: Thu, 1 Jul 2021 19:23:29 +0100 Subject: [PATCH 26/41] Linting. --- packages/server/src/api/controllers/row/ExternalRequest.ts | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/packages/server/src/api/controllers/row/ExternalRequest.ts b/packages/server/src/api/controllers/row/ExternalRequest.ts index 950cbef084..fcb6c6b75c 100644 --- a/packages/server/src/api/controllers/row/ExternalRequest.ts +++ b/packages/server/src/api/controllers/row/ExternalRequest.ts @@ -157,7 +157,6 @@ module External { // @ts-ignore const linkTablePrimary = linkTable.primary[0] if (!isMany(field)) { - newRow[field.foreignKey || linkTablePrimary] = breakRowIdField( row[key][0] )[0] @@ -166,9 +165,7 @@ module External { const isUpdate = !field.through const thisKey: string = isUpdate ? "id" : linkTablePrimary // @ts-ignore - const otherKey: string = isUpdate - ? field.foreignKey - : tablePrimary + const otherKey: string = isUpdate ? field.foreignKey : tablePrimary row[key].map((relationship: any) => { // we don't really support composite keys for relationships, this is why [0] is used manyRelationships.push({ From cae1a204114499c15ff2758cae561de9f6da8c78 Mon Sep 17 00:00:00 2001 From: Martin McKeaveney Date: Thu, 1 Jul 2021 21:24:23 +0100 Subject: [PATCH 27/41] preventing relationship overrides --- .../CreateEditRelationship.svelte | 113 ++++++------------ .../[selectedDatasource]/index.svelte | 9 +- packages/server/.vscode/launch.json | 9 -- packages/server/src/integrations/postgres.ts | 19 ++- 4 files changed, 58 insertions(+), 92 deletions(-) diff --git a/packages/builder/src/pages/builder/app/[application]/data/datasource/[selectedDatasource]/CreateEditRelationship/CreateEditRelationship.svelte b/packages/builder/src/pages/builder/app/[application]/data/datasource/[selectedDatasource]/CreateEditRelationship/CreateEditRelationship.svelte index a30c08c135..902d8fa4e2 100644 --- a/packages/builder/src/pages/builder/app/[application]/data/datasource/[selectedDatasource]/CreateEditRelationship/CreateEditRelationship.svelte +++ b/packages/builder/src/pages/builder/app/[application]/data/datasource/[selectedDatasource]/CreateEditRelationship/CreateEditRelationship.svelte @@ -8,10 +8,10 @@ export let from export let plusTables export let relationship = {} + export let close let originalName = relationship.name - $: console.log(relationship) $: tableOptions = plusTables.map(table => ({ label: table.name, value: table._id })) $: valid = relationship.name && relationship.tableId && relationship.relationshipType $: from = plusTables.find(table => table._id === relationship.source) @@ -19,18 +19,6 @@ $: through = plusTables.find(table => table._id === relationship.through) $: linkTable = through || to - $: relationshipOptions = from && to ? [ - { - name: `Many ${from.name} rows → many ${to.name} rows`, - alt: `Many ${from.name} rows → many ${to.name} rows`, - value: RelationshipTypes.MANY_TO_MANY, - }, - { - name: `One ${from.name} row → many ${to.name} rows`, - alt: `One ${from.name} rows → many ${to.name} rows`, - value: RelationshipTypes.ONE_TO_MANY, - } - ] : [] $: relationshipTypes = [ { @@ -51,25 +39,44 @@ // save the relationship on to the datasource async function saveRelationship() { + const manyToMany = relationship.relationshipType === RelationshipTypes.MANY_TO_MANY // source of relationship datasource.entities[from.name].schema[relationship.name] = { type: "link", ...relationship } - // if (originalName !== from.name) { - // delete datasource.entities[from.name].schema[originalName] - // } - // save other side of relationship in the other schema datasource.entities[to.name].schema[relationship.name] = { + name: relationship.name, type: "link", - relationshipType: relationship.relationshipType === RelationshipTypes.MANY_TO_MANY ? RelationshipTypes.MANY_TO_MANY : RelationshipTypes.MANY_TO_ONE, - tableId: to._id + relationshipType: manyToMany ? RelationshipTypes.MANY_TO_MANY : RelationshipTypes.MANY_TO_ONE, + tableId: from._id, + fieldName: relationship.fieldName, + foreignKey: relationship.foreignKey } + // If relationship has been renamed + if (originalName !== relationship.name) { + delete datasource.entities[from.name].schema[originalName] + delete datasource.entities[to.name].schema[originalName] + } + + console.log({ + from: datasource.entities[from.name].schema[relationship.name], + to: datasource.entities[to.name].schema[relationship.name], + }) + await save() await tables.fetch() } + + async function deleteRelationship() { + delete datasource.entities[from.name].schema[relationship.name] + delete datasource.entities[to.name].schema[relationship.name] + await save() + await tables.fetch() + close() + } {/if} - {#if relationship?.relationshipType === RelationshipTypes.ONE_TO_MANY} + {#if relationship?.relationshipType === RelationshipTypes.ONE_TO_MANY && to} + {/each} + \ No newline at end of file diff --git a/packages/server/src/constants/definitions.ts b/packages/server/src/constants/definitions.ts index b39f2550ab..8ab995adc4 100644 --- a/packages/server/src/constants/definitions.ts +++ b/packages/server/src/constants/definitions.ts @@ -37,6 +37,6 @@ export interface BudibaseAppMetadata { name: string url: string instance: { _id: string } - updatedAt: Date, + updatedAt: Date createdAt: Date -} \ No newline at end of file +} diff --git a/packages/server/src/integrations/postgres.ts b/packages/server/src/integrations/postgres.ts index 547f6bf049..f4f4e20582 100644 --- a/packages/server/src/integrations/postgres.ts +++ b/packages/server/src/integrations/postgres.ts @@ -135,10 +135,7 @@ module PostgresModule { * Fetches the tables from the postgres table and assigns them to the datasource. * @param {*} datasourceId - datasourceId to fetch */ - async buildSchema( - datasourceId: string, - entities: Record - ) { + async buildSchema(datasourceId: string, entities: Record) { let tableKeys: { [key: string]: string[] } = {} try { const primaryKeysResponse = await this.client.query( @@ -173,7 +170,7 @@ module PostgresModule { // add the existing relationships from the entities if they exist, to prevent them from being overridden if (entities) { - const existingTableSchema = entities[tableName].schema + const existingTableSchema = entities[tableName].schema for (let key in existingTableSchema) { if (existingTableSchema[key].type === "link") { tables[tableName].schema[key] = existingTableSchema[key] From a8ff2cc7bdd5c66643e2733517c8d5828e39a261 Mon Sep 17 00:00:00 2001 From: mike12345567 Date: Fri, 2 Jul 2021 14:35:05 +0100 Subject: [PATCH 29/41] Removing console logs and simplifying the relationship name. --- .../datasource/[selectedDatasource]/index.svelte | 12 ++++-------- 1 file changed, 4 insertions(+), 8 deletions(-) diff --git a/packages/builder/src/pages/builder/app/[application]/data/datasource/[selectedDatasource]/index.svelte b/packages/builder/src/pages/builder/app/[application]/data/datasource/[selectedDatasource]/index.svelte index 106a95f74c..756121b458 100644 --- a/packages/builder/src/pages/builder/app/[application]/data/datasource/[selectedDatasource]/index.svelte +++ b/packages/builder/src/pages/builder/app/[application]/data/datasource/[selectedDatasource]/index.svelte @@ -31,7 +31,6 @@ if (column.type !== "link") { continue } - console.log(`table - ${table.name} - ${column.name} - id: ${column._id} - ${column.main}`) // these relationships have an id to pair them to each other // one has a main for the from side const key = column.main ? "from" : "to" @@ -57,15 +56,12 @@ const fromTableName = getTableName(toCol.tableId) const toTableName = getTableName(fromCol.tableId) const throughTableName = getTableName(fromCol.through) - console.log(throughTableName) - let displayFrom = `${fromTableName} (${fromCol.name})` - let displayTo = `${toTableName} (${toCol.name})` let displayString if (throughTableName) { - displayString = `${displayFrom} through ${throughTableName} → ${displayTo}` + displayString = `${fromTableName} through ${throughTableName} → ${toTableName}` } else { - displayString = `${displayFrom} → ${displayTo}` + displayString = `${fromTableName} → ${toTableName}` } return displayString } @@ -219,13 +215,13 @@ on:click={() => openRelationshipModal(relationship.from, relationship.to)} > -

+

{buildRelationshipDisplayString( relationship.from, relationship.to )}

-

{relationship.from?.name} to {relationship.to?.name}

+

{relationship.from?.name} to {relationship.to?.name}

→

{/each} From 9aa672101e9483b3c15eb8493ffe7b3c358522ca Mon Sep 17 00:00:00 2001 From: mike12345567 Date: Fri, 2 Jul 2021 14:36:24 +0100 Subject: [PATCH 30/41] Linting. --- .../CreateEditRelationship.svelte | 60 +++++++++++-------- .../[selectedDatasource]/index.svelte | 2 +- .../modals/EditDisplayColumnsModal.svelte | 23 ++++--- 3 files changed, 51 insertions(+), 34 deletions(-) diff --git a/packages/builder/src/pages/builder/app/[application]/data/datasource/[selectedDatasource]/CreateEditRelationship/CreateEditRelationship.svelte b/packages/builder/src/pages/builder/app/[application]/data/datasource/[selectedDatasource]/CreateEditRelationship/CreateEditRelationship.svelte index f15d7b22f1..7ff1225d1c 100644 --- a/packages/builder/src/pages/builder/app/[application]/data/datasource/[selectedDatasource]/CreateEditRelationship/CreateEditRelationship.svelte +++ b/packages/builder/src/pages/builder/app/[application]/data/datasource/[selectedDatasource]/CreateEditRelationship/CreateEditRelationship.svelte @@ -11,31 +11,40 @@ export let toRelationship = {} export let close - let originalFromName = fromRelationship.name, originalToName = toRelationship.name + let originalFromName = fromRelationship.name, + originalToName = toRelationship.name function isValid(relationship) { - if (relationship.relationshipType === RelationshipTypes.MANY_TO_MANY && !relationship.through) { + if ( + relationship.relationshipType === RelationshipTypes.MANY_TO_MANY && + !relationship.through + ) { return false } - return relationship.name && relationship.tableId && relationship.relationshipType + return ( + relationship.name && relationship.tableId && relationship.relationshipType + ) } - $: tableOptions = plusTables.map(table => ({ label: table.name, value: table._id })) + $: tableOptions = plusTables.map(table => ({ + label: table.name, + value: table._id, + })) $: fromTable = plusTables.find(table => table._id === toRelationship?.tableId) $: toTable = plusTables.find(table => table._id === fromRelationship?.tableId) $: through = plusTables.find(table => table._id === fromRelationship?.through) $: valid = toTable && fromTable && isValid(fromRelationship) $: linkTable = through || toTable $: relationshipTypes = [ - { - label: "Many", - value: RelationshipTypes.MANY_TO_MANY, - }, - { - label: "One", - value: RelationshipTypes.MANY_TO_ONE, - } - ] + { + label: "Many", + value: RelationshipTypes.MANY_TO_MANY, + }, + { + label: "One", + value: RelationshipTypes.MANY_TO_ONE, + }, + ] $: updateRelationshipType(fromRelationship?.relationshipType) function updateRelationshipType(fromType) { @@ -48,7 +57,8 @@ function buildRelationships() { // if any to many only need to check from - const manyToMany = fromRelationship.relationshipType === RelationshipTypes.MANY_TO_MANY + const manyToMany = + fromRelationship.relationshipType === RelationshipTypes.MANY_TO_MANY // main is simply used to know this is the side the user configured it from const id = uuid() let relateFrom = { @@ -97,9 +107,11 @@ async function saveRelationship() { buildRelationships() // source of relationship - datasource.entities[fromTable.name].schema[fromRelationship.name] = fromRelationship + datasource.entities[fromTable.name].schema[fromRelationship.name] = + fromRelationship // save other side of relationship in the other schema - datasource.entities[toTable.name].schema[toRelationship.name] = toRelationship + datasource.entities[toTable.name].schema[toRelationship.name] = + toRelationship // If relationship has been renamed if (originalFromName !== fromRelationship.name) { @@ -139,35 +151,35 @@
- - {:else if toTable} {/each} - \ No newline at end of file + From fcf4dbc7a13ade288910e55066b6affdc5d03f19 Mon Sep 17 00:00:00 2001 From: mike12345567 Date: Fri, 2 Jul 2021 14:49:47 +0100 Subject: [PATCH 31/41] Updating server test cases. --- .../server/src/api/routes/tests/datasource.spec.js | 2 +- packages/server/src/integrations/tests/sql.spec.js | 10 +++++----- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/packages/server/src/api/routes/tests/datasource.spec.js b/packages/server/src/api/routes/tests/datasource.spec.js index d53001b06e..a041de4310 100644 --- a/packages/server/src/api/routes/tests/datasource.spec.js +++ b/packages/server/src/api/routes/tests/datasource.spec.js @@ -94,7 +94,7 @@ describe("/datasources", () => { .expect(200) // this is mock data, can't test it expect(res.body).toBeDefined() - expect(pg.queryMock).toHaveBeenCalledWith(`select "name", "age" from "users" where "name" like $1 limit $2`, ["John%", 5000]) + expect(pg.queryMock).toHaveBeenCalledWith(`select "name", "age" from "users" where "users"."name" like $1 limit $2`, ["John%", 5000]) }) }) diff --git a/packages/server/src/integrations/tests/sql.spec.js b/packages/server/src/integrations/tests/sql.spec.js index 2b6badd92d..fb57fe79e7 100644 --- a/packages/server/src/integrations/tests/sql.spec.js +++ b/packages/server/src/integrations/tests/sql.spec.js @@ -81,7 +81,7 @@ describe("SQL query builder", () => { })) expect(query).toEqual({ bindings: ["John%", limit], - sql: `select * from "${TABLE_NAME}" where "name" like $1 limit $2` + sql: `select * from "${TABLE_NAME}" where "${TABLE_NAME}"."name" like $1 limit $2` }) }) @@ -98,7 +98,7 @@ describe("SQL query builder", () => { })) expect(query).toEqual({ bindings: [2, 10, limit], - sql: `select * from "${TABLE_NAME}" where "age" between $1 and $2 limit $3` + sql: `select * from "${TABLE_NAME}" where "${TABLE_NAME}"."age" between $1 and $2 limit $3` }) }) @@ -114,7 +114,7 @@ describe("SQL query builder", () => { })) expect(query).toEqual({ bindings: [10, "John", limit], - sql: `select * from "${TABLE_NAME}" where ("age" = $1) or ("name" = $2) limit $3` + sql: `select * from "${TABLE_NAME}" where ("${TABLE_NAME}"."age" = $1) or ("${TABLE_NAME}"."name" = $2) limit $3` }) }) @@ -139,7 +139,7 @@ describe("SQL query builder", () => { })) expect(query).toEqual({ bindings: ["John", 1001], - sql: `update "${TABLE_NAME}" set "name" = $1 where "id" = $2 returning *` + sql: `update "${TABLE_NAME}" set "name" = $1 where "${TABLE_NAME}"."id" = $2 returning *` }) }) @@ -151,7 +151,7 @@ describe("SQL query builder", () => { })) expect(query).toEqual({ bindings: [1001], - sql: `delete from "${TABLE_NAME}" where "id" = $1 returning *` + sql: `delete from "${TABLE_NAME}" where "${TABLE_NAME}"."id" = $1 returning *` }) }) From c24cc5c3fb1863bb01eb49cdc2a9514f872b1243 Mon Sep 17 00:00:00 2001 From: mike12345567 Date: Fri, 2 Jul 2021 18:42:01 +0100 Subject: [PATCH 32/41] Fixing issue with many to many through junction table not realising some exist, or some need deleted - as well as removing limit from details screen, this was blocking join statements and served no purpose (its already a search by equals). --- .../store/screenTemplates/rowDetailScreen.js | 1 - .../api/controllers/row/ExternalRequest.ts | 122 +++++++++++++++--- 2 files changed, 101 insertions(+), 22 deletions(-) diff --git a/packages/builder/src/builderStore/store/screenTemplates/rowDetailScreen.js b/packages/builder/src/builderStore/store/screenTemplates/rowDetailScreen.js index b41b4085a8..cb97bd53db 100644 --- a/packages/builder/src/builderStore/store/screenTemplates/rowDetailScreen.js +++ b/packages/builder/src/builderStore/store/screenTemplates/rowDetailScreen.js @@ -98,7 +98,6 @@ const createScreen = table => { valueType: "Binding", }, ], - limit: 1, paginate: false, }) diff --git a/packages/server/src/api/controllers/row/ExternalRequest.ts b/packages/server/src/api/controllers/row/ExternalRequest.ts index fcb6c6b75c..7e1ab52e83 100644 --- a/packages/server/src/api/controllers/row/ExternalRequest.ts +++ b/packages/server/src/api/controllers/row/ExternalRequest.ts @@ -33,6 +33,7 @@ module External { const { breakExternalTableId } = require("../../../integrations/utils") const { processObjectSync } = require("@budibase/string-templates") const { cloneDeep } = require("lodash/fp") + const { isEqual } = require("lodash") function buildFilters( id: string | undefined, @@ -93,6 +94,18 @@ module External { return generateRowIdField(idParts) } + function getEndpoint(tableId: string | undefined, operation: string) { + if (!tableId) { + return {} + } + const { datasourceId, tableName } = breakExternalTableId(tableId) + return { + datasourceId, + entityId: tableName, + operation, + } + } + function basicProcessing(row: Row, table: Table) { const thisRow: { [key: string]: any } = {} // filter the row down to what is actually the row (not joined) @@ -112,7 +125,7 @@ module External { } class ExternalRequest { - private appId: string + private readonly appId: string private operation: Operation private tableId: string private tables: { [key: string]: Table } @@ -173,7 +186,7 @@ module External { isUpdate, [thisKey]: breakRowIdField(relationship)[0], // leave the ID for enrichment later - [otherKey]: `{{ ${tablePrimary} }}`, + [otherKey]: `{{ literal ${tablePrimary} }}`, }) }) } @@ -184,6 +197,11 @@ module External { return { row: newRow, manyRelationships } } + /** + * This iterates through the returned rows and works out what elements of the rows + * actually match up to another row (based on primary keys) - this is pretty specific + * to SQL and the way that SQL relationships are returned based on joins. + */ updateRelationshipColumns( row: Row, rows: { [key: string]: Row }, @@ -260,6 +278,11 @@ module External { return Object.values(finalRows) } + /** + * Gets the list of relationship JSON structures based on the columns in the table, + * this will be used by the underlying library to build whatever relationship mechanism + * it has (e.g. SQL joins). + */ buildRelationships(table: Table): RelationshipsJson[] { const relationships = [] for (let [fieldName, field] of Object.entries(table.schema)) { @@ -298,31 +321,88 @@ module External { return relationships } + /** + * This is a cached lookup, of relationship records, this is mainly for creating/deleting junction + * information. + */ + async lookup(row: Row, relationship: ManyRelationship, cache: {[key: string]: Row[]} = {}) { + const { tableId, isUpdate, id, ...rest } = relationship + const { tableName } = breakExternalTableId(tableId) + const table = this.tables[tableName] + if (isUpdate) { + return { rows: [], table } + } + // if not updating need to make sure we have a list of all possible options + let fullKey: string = tableId + "/", rowKey: string = "" + for (let key of Object.keys(rest)) { + if (row[key]) { + fullKey += key + rowKey = key + } + } + if (cache[fullKey] == null) { + cache[fullKey] = await makeExternalQuery(this.appId, { + endpoint: getEndpoint(tableId, DataSourceOperation.READ), + filters: { + equal: { + [rowKey]: row[rowKey], + }, + }, + }) + } + return { rows: cache[fullKey], table } + } + + /** + * Once a row has been written we may need to update a many field, e.g. updating foreign keys + * in a bunch of rows in another table, or inserting/deleting rows from a junction table (many to many). + * This is quite a complex process and is handled by this function, there are a few things going on here: + * 1. If updating foreign keys its relatively simple, just create a filter for the row that needs updated + * and write the various components. + * 2. If junction table, then we lookup what exists already, write what doesn't exist, work out what + * isn't supposed to exist anymore and delete those. This is better than the usual method of delete them + * all and then re-create, as theres no chance of losing data (e.g. delete succeed, but write fail). + */ async handleManyRelationships(row: Row, relationships: ManyRelationship[]) { - const { appId, tables } = this + const { appId } = this + if (relationships.length === 0) { + return + } + // if we're creating (in a through table) need to wipe the existing ones first const promises = [] + const cache: { [key:string]: Row[] } = {} for (let relationship of relationships) { const { tableId, isUpdate, id, ...rest } = relationship - const { datasourceId, tableName } = breakExternalTableId(tableId) - const linkedTable = tables[tableName] - if (!linkedTable) { - continue + const body = processObjectSync(rest, row) + const { table, rows } = await this.lookup(row, relationship, cache) + const found = rows.find(row => isEqual(body, row)) + const operation = isUpdate ? DataSourceOperation.UPDATE : DataSourceOperation.CREATE + if (!found) { + promises.push( + makeExternalQuery(appId, { + endpoint: getEndpoint(tableId, operation), + // if we're doing many relationships then we're writing, only one response + body, + filters: buildFilters(id, {}, table), + }) + ) + } else { + // remove the relationship from the rows + rows.splice(rows.indexOf(found), 1) } - const endpoint = { - datasourceId, - entityId: tableName, - operation: isUpdate - ? DataSourceOperation.UPDATE - : DataSourceOperation.CREATE, + } + // finally if creating, cleanup any rows that aren't supposed to be here + for (let [key, rows] of Object.entries(cache)) { + // @ts-ignore + const tableId: string = key.split("/").shift() + const { tableName } = breakExternalTableId(tableId) + const table = this.tables[tableName] + for (let row of rows) { + promises.push(makeExternalQuery(this.appId, { + endpoint: getEndpoint(tableId, DataSourceOperation.DELETE), + filters: buildFilters(generateIdForRow(row, table), {}, table) + })) } - promises.push( - makeExternalQuery(appId, { - endpoint, - // if we're doing many relationships then we're writing, only one response - body: processObjectSync(rest, row), - filters: buildFilters(id, {}, linkedTable), - }) - ) } await Promise.all(promises) } From 2a1243e81e07debdd8cb34d8e2e590547a9c1528 Mon Sep 17 00:00:00 2001 From: mike12345567 Date: Fri, 2 Jul 2021 18:44:01 +0100 Subject: [PATCH 33/41] Linting. --- .../api/controllers/row/ExternalRequest.ts | 25 +++++++++++++------ 1 file changed, 17 insertions(+), 8 deletions(-) diff --git a/packages/server/src/api/controllers/row/ExternalRequest.ts b/packages/server/src/api/controllers/row/ExternalRequest.ts index 7e1ab52e83..876ed18ff0 100644 --- a/packages/server/src/api/controllers/row/ExternalRequest.ts +++ b/packages/server/src/api/controllers/row/ExternalRequest.ts @@ -325,7 +325,11 @@ module External { * This is a cached lookup, of relationship records, this is mainly for creating/deleting junction * information. */ - async lookup(row: Row, relationship: ManyRelationship, cache: {[key: string]: Row[]} = {}) { + async lookup( + row: Row, + relationship: ManyRelationship, + cache: { [key: string]: Row[] } = {} + ) { const { tableId, isUpdate, id, ...rest } = relationship const { tableName } = breakExternalTableId(tableId) const table = this.tables[tableName] @@ -333,7 +337,8 @@ module External { return { rows: [], table } } // if not updating need to make sure we have a list of all possible options - let fullKey: string = tableId + "/", rowKey: string = "" + let fullKey: string = tableId + "/", + rowKey: string = "" for (let key of Object.keys(rest)) { if (row[key]) { fullKey += key @@ -370,13 +375,15 @@ module External { } // if we're creating (in a through table) need to wipe the existing ones first const promises = [] - const cache: { [key:string]: Row[] } = {} + const cache: { [key: string]: Row[] } = {} for (let relationship of relationships) { const { tableId, isUpdate, id, ...rest } = relationship const body = processObjectSync(rest, row) const { table, rows } = await this.lookup(row, relationship, cache) const found = rows.find(row => isEqual(body, row)) - const operation = isUpdate ? DataSourceOperation.UPDATE : DataSourceOperation.CREATE + const operation = isUpdate + ? DataSourceOperation.UPDATE + : DataSourceOperation.CREATE if (!found) { promises.push( makeExternalQuery(appId, { @@ -398,10 +405,12 @@ module External { const { tableName } = breakExternalTableId(tableId) const table = this.tables[tableName] for (let row of rows) { - promises.push(makeExternalQuery(this.appId, { - endpoint: getEndpoint(tableId, DataSourceOperation.DELETE), - filters: buildFilters(generateIdForRow(row, table), {}, table) - })) + promises.push( + makeExternalQuery(this.appId, { + endpoint: getEndpoint(tableId, DataSourceOperation.DELETE), + filters: buildFilters(generateIdForRow(row, table), {}, table), + }) + ) } } await Promise.all(promises) From 71f96f9601bba39cc50d139b92c65d8079ea489e Mon Sep 17 00:00:00 2001 From: mike12345567 Date: Sat, 3 Jul 2021 10:26:37 +0100 Subject: [PATCH 34/41] Fixing little UI bugs. --- .../datasource/[selectedDatasource]/index.svelte | 12 +++++++++--- packages/server/src/integrations/postgres.ts | 6 +++++- 2 files changed, 14 insertions(+), 4 deletions(-) diff --git a/packages/builder/src/pages/builder/app/[application]/data/datasource/[selectedDatasource]/index.svelte b/packages/builder/src/pages/builder/app/[application]/data/datasource/[selectedDatasource]/index.svelte index 9905d80684..85f300119c 100644 --- a/packages/builder/src/pages/builder/app/[application]/data/datasource/[selectedDatasource]/index.svelte +++ b/packages/builder/src/pages/builder/app/[application]/data/datasource/[selectedDatasource]/index.svelte @@ -177,9 +177,11 @@ Update display columns {/if} - +
+ +
@@ -319,4 +321,8 @@ grid-gap: var(--spacing-l); grid-template-columns: 1fr 1fr; } + + .table-buttons div { + grid-column-end: -1; + } diff --git a/packages/server/src/integrations/postgres.ts b/packages/server/src/integrations/postgres.ts index f4f4e20582..935bfbeeea 100644 --- a/packages/server/src/integrations/postgres.ts +++ b/packages/server/src/integrations/postgres.ts @@ -134,6 +134,7 @@ module PostgresModule { /** * Fetches the tables from the postgres table and assigns them to the datasource. * @param {*} datasourceId - datasourceId to fetch + * @param entities - the tables that are to be built */ async buildSchema(datasourceId: string, entities: Record) { let tableKeys: { [key: string]: string[] } = {} @@ -169,9 +170,12 @@ module PostgresModule { } // add the existing relationships from the entities if they exist, to prevent them from being overridden - if (entities) { + if (entities && entities[tableName]) { const existingTableSchema = entities[tableName].schema for (let key in existingTableSchema) { + if (!existingTableSchema.hasOwnProperty(key)) { + continue + } if (existingTableSchema[key].type === "link") { tables[tableName].schema[key] = existingTableSchema[key] } From 11f6abee0aa764cd9f13065a9286303eccce308b Mon Sep 17 00:00:00 2001 From: mike12345567 Date: Sat, 3 Jul 2021 11:15:01 +0100 Subject: [PATCH 35/41] Re-writing a bit so that it is aware some functionality is SQL only, makes future plus endpoints easier. --- .../api/controllers/row/ExternalRequest.ts | 30 ++++++++----- .../src/api/controllers/row/external.js | 12 +++-- packages/server/src/definitions/common.ts | 16 +++++++ packages/server/src/definitions/datasource.ts | 14 ++++++ packages/server/src/integrations/index.ts | 45 ++++++++++--------- packages/server/src/integrations/utils.ts | 10 +++++ 6 files changed, 92 insertions(+), 35 deletions(-) diff --git a/packages/server/src/api/controllers/row/ExternalRequest.ts b/packages/server/src/api/controllers/row/ExternalRequest.ts index 876ed18ff0..7e9f7d7077 100644 --- a/packages/server/src/api/controllers/row/ExternalRequest.ts +++ b/packages/server/src/api/controllers/row/ExternalRequest.ts @@ -5,7 +5,7 @@ import { PaginationJson, RelationshipsJson, } from "../../../definitions/datasource" -import { Row, Table, FieldSchema } from "../../../definitions/common" +import {Row, Table, FieldSchema, Datasource} from "../../../definitions/common" import { breakRowIdField, generateRowIdField, @@ -29,11 +29,11 @@ interface RunConfig { module External { const { makeExternalQuery } = require("./utils") const { DataSourceOperation, FieldTypes } = require("../../../constants") - const { getAllExternalTables } = require("../table/utils") - const { breakExternalTableId } = require("../../../integrations/utils") + const { breakExternalTableId, isSQL } = require("../../../integrations/utils") const { processObjectSync } = require("@budibase/string-templates") const { cloneDeep } = require("lodash/fp") const { isEqual } = require("lodash") + const CouchDB = require("../../../db") function buildFilters( id: string | undefined, @@ -128,18 +128,22 @@ module External { private readonly appId: string private operation: Operation private tableId: string - private tables: { [key: string]: Table } + private datasource: Datasource + private tables: { [key: string]: Table } = {} constructor( appId: string, operation: Operation, tableId: string, - tables: { [key: string]: Table } + datasource: Datasource ) { this.appId = appId this.operation = operation this.tableId = tableId - this.tables = tables + this.datasource = datasource + if (datasource && datasource.entities) { + this.tables = datasource.entities + } } inputProcessing(row: Row, table: Table) { @@ -451,10 +455,16 @@ module External { async run({ id, row, filters, sort, paginate }: RunConfig) { const { appId, operation, tableId } = this let { datasourceId, tableName } = breakExternalTableId(tableId) - if (!this.tables) { - this.tables = await getAllExternalTables(appId, datasourceId) + if (!this.datasource) { + const db = new CouchDB(appId) + this.datasource = await db.get(datasourceId) + if (!this.datasource || !this.datasource.entities) { + throw "No tables found, fetch tables before query." + } + this.tables = this.datasource.entities } const table = this.tables[tableName] + let isSql = isSQL(this.datasource) if (!table) { throw `Unable to process query, table "${tableName}" not defined.` } @@ -476,8 +486,8 @@ module External { operation, }, resource: { - // have to specify the fields to avoid column overlap - fields: this.buildFields(table), + // have to specify the fields to avoid column overlap (for SQL) + fields: isSql ? this.buildFields(table) : [], }, filters, sort, diff --git a/packages/server/src/api/controllers/row/external.js b/packages/server/src/api/controllers/row/external.js index 96928fb2ae..9f0a55e2cd 100644 --- a/packages/server/src/api/controllers/row/external.js +++ b/packages/server/src/api/controllers/row/external.js @@ -9,9 +9,10 @@ const { breakRowIdField, } = require("../../../integrations/utils") const ExternalRequest = require("./ExternalRequest") +const CouchDB = require("../../../db") async function handleRequest(appId, operation, tableId, opts = {}) { - return new ExternalRequest(appId, operation, tableId, opts.tables).run(opts) + return new ExternalRequest(appId, operation, tableId, opts.datasource).run(opts) } exports.patch = async ctx => { @@ -162,14 +163,19 @@ exports.fetchEnrichedRow = async ctx => { const id = ctx.params.rowId const tableId = ctx.params.tableId const { datasourceId, tableName } = breakExternalTableId(tableId) - const tables = await getAllExternalTables(appId, datasourceId) + const db = new CouchDB(appId) + const datasource = await db.get(datasourceId) + if (!datasource || !datasource.entities) { + ctx.throw(400, "Datasource has not been configured for plus API.") + } + const tables = datasource.entities const response = await handleRequest( appId, DataSourceOperation.READ, tableId, { id, - tables, + datasource, } ) const table = tables[tableName] diff --git a/packages/server/src/definitions/common.ts b/packages/server/src/definitions/common.ts index 0115cc7685..497f8f68f2 100644 --- a/packages/server/src/definitions/common.ts +++ b/packages/server/src/definitions/common.ts @@ -1,3 +1,5 @@ +import { SourceNames } from "./datasource" + interface Base { _id?: string _rev?: string @@ -82,3 +84,17 @@ export interface Automation extends Base { trigger?: AutomationStep } } + +export interface Datasource extends Base { + type: string + name: string + source: SourceNames + // the config is defined by the schema + config: { + [key: string]: string | number | boolean + } + plus: boolean + entities?: { + [key: string]: Table + } +} diff --git a/packages/server/src/definitions/datasource.ts b/packages/server/src/definitions/datasource.ts index ac52a32026..22f1998601 100644 --- a/packages/server/src/definitions/datasource.ts +++ b/packages/server/src/definitions/datasource.ts @@ -26,6 +26,20 @@ export enum DatasourceFieldTypes { JSON = "json", } +export enum SourceNames { + POSTGRES = "POSTGRES", + DYNAMODB = "DYNAMODB", + MONGODB = "MONGODB", + ELASTICSEARCH = "ELASTICSEARCH", + COUCHDB = "COUCHDB", + SQL_SERVER = "SQL_SERVER", + S3 = "S3", + AIRTABLE = "AIRTABLE", + MYSQL = "MYSQL", + ARANGODB = "ARANGODB", + REST = "REST", +} + export interface QueryDefinition { type: QueryTypes displayName?: string diff --git a/packages/server/src/integrations/index.ts b/packages/server/src/integrations/index.ts index 4999f0c867..c0acd6b225 100644 --- a/packages/server/src/integrations/index.ts +++ b/packages/server/src/integrations/index.ts @@ -9,33 +9,34 @@ const airtable = require("./airtable") const mysql = require("./mysql") const arangodb = require("./arangodb") const rest = require("./rest") +const { SourceNames } = require("../definitions/datasource") const DEFINITIONS = { - POSTGRES: postgres.schema, - DYNAMODB: dynamodb.schema, - MONGODB: mongodb.schema, - ELASTICSEARCH: elasticsearch.schema, - COUCHDB: couchdb.schema, - SQL_SERVER: sqlServer.schema, - S3: s3.schema, - AIRTABLE: airtable.schema, - MYSQL: mysql.schema, - ARANGODB: arangodb.schema, - REST: rest.schema, + [SourceNames.POSTGRES]: postgres.schema, + [SourceNames.DYNAMODB]: dynamodb.schema, + [SourceNames.MONGODB]: mongodb.schema, + [SourceNames.ELASTICSEARCH]: elasticsearch.schema, + [SourceNames.COUCHDB]: couchdb.schema, + [SourceNames.SQL_SERVER]: sqlServer.schema, + [SourceNames.S3]: s3.schema, + [SourceNames.AIRTABLE]: airtable.schema, + [SourceNames.MYSQL]: mysql.schema, + [SourceNames.ARANGODB]: arangodb.schema, + [SourceNames.REST]: rest.schema, } const INTEGRATIONS = { - POSTGRES: postgres.integration, - DYNAMODB: dynamodb.integration, - MONGODB: mongodb.integration, - ELASTICSEARCH: elasticsearch.integration, - COUCHDB: couchdb.integration, - S3: s3.integration, - SQL_SERVER: sqlServer.integration, - AIRTABLE: airtable.integration, - MYSQL: mysql.integration, - ARANGODB: arangodb.integration, - REST: rest.integration, + [SourceNames.POSTGRES]: postgres.integration, + [SourceNames.DYNAMODB]: dynamodb.integration, + [SourceNames.MONGODB]: mongodb.integration, + [SourceNames.ELASTICSEARCH]: elasticsearch.integration, + [SourceNames.COUCHDB]: couchdb.integration, + [SourceNames.SQL_SERVER]: s3.integration, + [SourceNames.S3]: sqlServer.integration, + [SourceNames.AIRTABLE]: airtable.integration, + [SourceNames.MYSQL]: mysql.integration, + [SourceNames.ARANGODB]: arangodb.integration, + [SourceNames.REST]: rest.integration, } module.exports = { diff --git a/packages/server/src/integrations/utils.ts b/packages/server/src/integrations/utils.ts index b8e9ee1dda..9140764094 100644 --- a/packages/server/src/integrations/utils.ts +++ b/packages/server/src/integrations/utils.ts @@ -1,4 +1,6 @@ import { SqlQuery } from "../definitions/datasource" +import { Datasource } from "../definitions/common" +import { SourceNames } from "../definitions/datasource" const { DocumentTypes, SEPARATOR } = require("../db/utils") const { FieldTypes } = require("../constants") @@ -51,3 +53,11 @@ export function getSqlQuery(query: SqlQuery | string): SqlQuery { return query } } + +export function isSQL(datasource: Datasource): boolean { + if (!datasource || !datasource.source) { + return false + } + const SQL = [SourceNames.POSTGRES, SourceNames.SQL_SERVER, SourceNames.MYSQL] + return SQL.indexOf(datasource.source) !== -1 +} From 3849d2a968bd34d2e7747d36e8c5a8da6ce0ca6c Mon Sep 17 00:00:00 2001 From: mike12345567 Date: Sat, 3 Jul 2021 11:16:29 +0100 Subject: [PATCH 36/41] Linting. --- packages/server/src/api/controllers/row/ExternalRequest.ts | 7 ++++++- packages/server/src/api/controllers/row/external.js | 5 +++-- 2 files changed, 9 insertions(+), 3 deletions(-) diff --git a/packages/server/src/api/controllers/row/ExternalRequest.ts b/packages/server/src/api/controllers/row/ExternalRequest.ts index 7e9f7d7077..855c64e4c1 100644 --- a/packages/server/src/api/controllers/row/ExternalRequest.ts +++ b/packages/server/src/api/controllers/row/ExternalRequest.ts @@ -5,7 +5,12 @@ import { PaginationJson, RelationshipsJson, } from "../../../definitions/datasource" -import {Row, Table, FieldSchema, Datasource} from "../../../definitions/common" +import { + Row, + Table, + FieldSchema, + Datasource, +} from "../../../definitions/common" import { breakRowIdField, generateRowIdField, diff --git a/packages/server/src/api/controllers/row/external.js b/packages/server/src/api/controllers/row/external.js index 9f0a55e2cd..3a96064a9f 100644 --- a/packages/server/src/api/controllers/row/external.js +++ b/packages/server/src/api/controllers/row/external.js @@ -3,7 +3,6 @@ const { SortDirection, FieldTypes, } = require("../../../constants") -const { getAllExternalTables } = require("../table/utils") const { breakExternalTableId, breakRowIdField, @@ -12,7 +11,9 @@ const ExternalRequest = require("./ExternalRequest") const CouchDB = require("../../../db") async function handleRequest(appId, operation, tableId, opts = {}) { - return new ExternalRequest(appId, operation, tableId, opts.datasource).run(opts) + return new ExternalRequest(appId, operation, tableId, opts.datasource).run( + opts + ) } exports.patch = async ctx => { From 57278b4532caef8cd0a517084ad175c29953005d Mon Sep 17 00:00:00 2001 From: mike12345567 Date: Mon, 5 Jul 2021 09:29:16 +0100 Subject: [PATCH 37/41] Allowing user firstname and lastname to be updated through user portal. --- .../builder/portal/manage/users/[userId].svelte | 14 ++++++++++++-- 1 file changed, 12 insertions(+), 2 deletions(-) diff --git a/packages/builder/src/pages/builder/portal/manage/users/[userId].svelte b/packages/builder/src/pages/builder/portal/manage/users/[userId].svelte index ac5b569411..495e45e7c3 100644 --- a/packages/builder/src/pages/builder/portal/manage/users/[userId].svelte +++ b/packages/builder/src/pages/builder/portal/manage/users/[userId].svelte @@ -60,6 +60,16 @@ let toggleDisabled = false + async function updateUserFirstName(evt) { + await users.save({ ...$userFetch?.data, firstName: evt.target.value }) + await userFetch.refresh() + } + + async function updateUserLastName(evt) { + await users.save({ ...$userFetch?.data, lastName: evt.target.value }) + await userFetch.refresh() + } + async function toggleFlag(flagName, detail) { toggleDisabled = true await users.save({ ...$userFetch?.data, [flagName]: { global: detail } }) @@ -113,11 +123,11 @@
- +
- +
{#if userId !== $auth.user._id} From f450e38daba79d5f3d81a43b359b4f17b93d964b Mon Sep 17 00:00:00 2001 From: mike12345567 Date: Mon, 5 Jul 2021 09:38:01 +0100 Subject: [PATCH 38/41] Linting. --- .../builder/portal/manage/users/[userId].svelte | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/packages/builder/src/pages/builder/portal/manage/users/[userId].svelte b/packages/builder/src/pages/builder/portal/manage/users/[userId].svelte index 495e45e7c3..912506d0cf 100644 --- a/packages/builder/src/pages/builder/portal/manage/users/[userId].svelte +++ b/packages/builder/src/pages/builder/portal/manage/users/[userId].svelte @@ -123,11 +123,19 @@
- +
- +
{#if userId !== $auth.user._id} From ea8fea98a97c4dfa8bc48edde99012fc90203c63 Mon Sep 17 00:00:00 2001 From: mike12345567 Date: Mon, 5 Jul 2021 10:48:28 +0100 Subject: [PATCH 39/41] Re-working UI after some review. --- .../CreateEditRelationship.svelte | 78 +++++++------------ 1 file changed, 29 insertions(+), 49 deletions(-) diff --git a/packages/builder/src/pages/builder/app/[application]/data/datasource/[selectedDatasource]/CreateEditRelationship/CreateEditRelationship.svelte b/packages/builder/src/pages/builder/app/[application]/data/datasource/[selectedDatasource]/CreateEditRelationship/CreateEditRelationship.svelte index 7ff1225d1c..e1546dd0b1 100644 --- a/packages/builder/src/pages/builder/app/[application]/data/datasource/[selectedDatasource]/CreateEditRelationship/CreateEditRelationship.svelte +++ b/packages/builder/src/pages/builder/app/[application]/data/datasource/[selectedDatasource]/CreateEditRelationship/CreateEditRelationship.svelte @@ -1,6 +1,6 @@ -
-
- -
-
- -
+ - - + + {#if fromRelationship?.relationshipType === RelationshipTypes.MANY_TO_MANY} + + {:else if toTable} + +
{#if originalFromName !== null} @@ -192,19 +184,7 @@ From 006db17bfc34fcc0b9881a51599da121daf7a705 Mon Sep 17 00:00:00 2001 From: mike12345567 Date: Mon, 5 Jul 2021 11:03:54 +0100 Subject: [PATCH 40/41] Quick fixes for UI, making sure relationships UI not displayed unless tables exist and fixing issue with through being sent up with one-to-many. --- .../CreateEditRelationship.svelte | 22 +++++++++++-------- .../[selectedDatasource]/index.svelte | 21 +++++++++--------- 2 files changed, 24 insertions(+), 19 deletions(-) diff --git a/packages/builder/src/pages/builder/app/[application]/data/datasource/[selectedDatasource]/CreateEditRelationship/CreateEditRelationship.svelte b/packages/builder/src/pages/builder/app/[application]/data/datasource/[selectedDatasource]/CreateEditRelationship/CreateEditRelationship.svelte index e1546dd0b1..ab5580cd88 100644 --- a/packages/builder/src/pages/builder/app/[application]/data/datasource/[selectedDatasource]/CreateEditRelationship/CreateEditRelationship.svelte +++ b/packages/builder/src/pages/builder/app/[application]/data/datasource/[selectedDatasource]/CreateEditRelationship/CreateEditRelationship.svelte @@ -61,6 +61,10 @@ fromRelationship.relationshipType === RelationshipTypes.MANY_TO_MANY // main is simply used to know this is the side the user configured it from const id = uuid() + if (!manyToMany) { + delete fromRelationship.through + delete toRelationship.through + } let relateFrom = { ...fromRelationship, type: "link", @@ -154,21 +158,21 @@ bind:value={toRelationship.tableId} /> {:else if toTable}
- {#if originalFromName !== null} + {#if originalFromName != null} {/if}
diff --git a/packages/builder/src/pages/builder/app/[application]/data/datasource/[selectedDatasource]/index.svelte b/packages/builder/src/pages/builder/app/[application]/data/datasource/[selectedDatasource]/index.svelte index 790a97c4d2..b0377d2f27 100644 --- a/packages/builder/src/pages/builder/app/[application]/data/datasource/[selectedDatasource]/index.svelte +++ b/packages/builder/src/pages/builder/app/[application]/data/datasource/[selectedDatasource]/index.svelte @@ -179,7 +179,7 @@ {/if}
@@ -203,12 +203,12 @@
Relationships Create relationship
- Tell budibase how your tables are related to get even more smart - features. + Tell budibase how your tables are related to get even more smart + features. {/if}