From 46c14fafc5da7f032a8dfcb199fff269e54c97b7 Mon Sep 17 00:00:00 2001 From: Martin McKeaveney Date: Wed, 23 Jun 2021 14:29:40 +0100 Subject: [PATCH 01/62] started on opinionated relationships --- .../CreateEditRelationship.svelte | 79 +++++++++++++++++++ .../[selectedDatasource]/index.svelte | 61 +++++++++++--- 2 files changed, 127 insertions(+), 13 deletions(-) create mode 100644 packages/builder/src/pages/builder/app/[application]/data/datasource/[selectedDatasource]/CreateEditRelationship.svelte diff --git a/packages/builder/src/pages/builder/app/[application]/data/datasource/[selectedDatasource]/CreateEditRelationship.svelte b/packages/builder/src/pages/builder/app/[application]/data/datasource/[selectedDatasource]/CreateEditRelationship.svelte new file mode 100644 index 0000000000..b9f6dbf092 --- /dev/null +++ b/packages/builder/src/pages/builder/app/[application]/data/datasource/[selectedDatasource]/CreateEditRelationship.svelte @@ -0,0 +1,79 @@ + + + + {#if step === 0} + Select your table + {:else if step === 1} + + Step 2 + {:else if step === 2} + Step 3 + {/if} + + + \ No newline at end of file diff --git a/packages/builder/src/pages/builder/app/[application]/data/datasource/[selectedDatasource]/index.svelte b/packages/builder/src/pages/builder/app/[application]/data/datasource/[selectedDatasource]/index.svelte index f805a5724f..b5a084120e 100644 --- a/packages/builder/src/pages/builder/app/[application]/data/datasource/[selectedDatasource]/index.svelte +++ b/packages/builder/src/pages/builder/app/[application]/data/datasource/[selectedDatasource]/index.svelte @@ -1,16 +1,19 @@ + + + + {#if datasource && integration}
@@ -100,19 +111,43 @@ having to write any queries at all.
- {#if datasource.entities} - {#each Object.keys(datasource.entities) as entity} -
onClickTable(datasource.entities[entity])} - > -

{entity}

-

Primary Key: {datasource.entities[entity].primary}

-

→

-
- {/each} - {/if} + {#each plusTables as table} +
onClickTable(table)} + > +

{table.name}

+

Primary Key: {table.primary}

+

→

+
+ {/each}
+ + +
+ Relationships + +
+ + Tell budibase how your tables are related to get even more smart features. + +
+ {#each plusTables as table} + {#each Object.keys(table) as column} + {#if table[column].type === "link"} +
onClickTable(table[column])} + > +

{table[column].name}

+

Primary Key: {table[column].primary}

+

→

+
+ {/if} + {/each} + {/each} +
+ {/if}
From 8a84d5cd6f116fd9140764557a6d2c7277fe2a84 Mon Sep 17 00:00:00 2001 From: mike12345567 Date: Wed, 23 Jun 2021 19:05:32 +0100 Subject: [PATCH 02/62] WIP - basic override of foreign keys. --- .../src/api/controllers/row/external.js | 85 ++++++++++++++++--- .../server/src/api/controllers/table/utils.js | 11 ++- packages/server/src/integrations/base/sql.js | 26 +++++- packages/server/src/integrations/postgres.js | 14 +++ 4 files changed, 117 insertions(+), 19 deletions(-) diff --git a/packages/server/src/api/controllers/row/external.js b/packages/server/src/api/controllers/row/external.js index 896f5a78e2..79a347750f 100644 --- a/packages/server/src/api/controllers/row/external.js +++ b/packages/server/src/api/controllers/row/external.js @@ -1,6 +1,6 @@ const { makeExternalQuery } = require("./utils") -const { DataSourceOperation, SortDirection } = require("../../../constants") -const { getExternalTable } = require("../table/utils") +const { DataSourceOperation, SortDirection, FieldTypes, RelationshipTypes } = require("../../../constants") +const { getAllExternalTables } = require("../table/utils") const { breakExternalTableId, generateRowIdField, @@ -35,17 +35,56 @@ function generateIdForRow(row, table) { return generateRowIdField(idParts) } -function outputProcessing(rows, table) { +function updateRelationshipColumns(rows, row, relationships, allTables) { + const columns = {} + for (let relationship of relationships) { + const linkedTable = allTables[relationship.tableName] + if (!linkedTable) { + continue + } + const display = linkedTable.primaryDisplay + const related = {} + if (display && row[display]) { + related.primaryDisplay = row[display] + } + related._id = row[relationship.to] + columns[relationship.from] = related + } + for (let [column, related] of Object.entries(columns)) { + if (!Array.isArray(rows[row._id][column])) { + rows[row._id][column] = [] + } + rows[row._id][column].push(related) + } + return rows +} + +function outputProcessing(rows, table, relationships, allTables) { // if no rows this is what is returned? Might be PG only if (rows[0].read === true) { return [] } + let finalRows = {} for (let row of rows) { row._id = generateIdForRow(row, table) - row.tableId = table._id - row._rev = "rev" + // this is a relationship of some sort + if (finalRows[row._id]) { + finalRows = updateRelationshipColumns(finalRows, row, relationships, allTables) + continue + } + const thisRow = {} + // filter the row down to what is actually the row (not joined) + for (let fieldName of Object.keys(table.schema)) { + thisRow[fieldName] = row[fieldName] + } + thisRow._id = row._id + thisRow.tableId = table._id + thisRow._rev = "rev" + finalRows[thisRow._id] = thisRow + // do this at end once its been added to the final rows + finalRows = updateRelationshipColumns(finalRows, row, relationships, allTables) } - return rows + return Object.values(finalRows) } function buildFilters(id, filters, table) { @@ -83,6 +122,26 @@ function buildFilters(id, filters, table) { } } +function buildRelationships(table) { + const relationships = [] + for (let [fieldName, field] of Object.entries(table.schema)) { + if (field.type !== FieldTypes.LINK) { + continue + } + // TODO: through field + if (field.relationshipType === RelationshipTypes.MANY_TO_MANY) { + continue + } + const broken = breakExternalTableId(field.tableId) + relationships.push({ + from: fieldName, + to: field.fieldName, + tableName: broken.tableName, + }) + } + return relationships +} + async function handleRequest( appId, operation, @@ -90,12 +149,14 @@ async function handleRequest( { id, row, filters, sort, paginate } = {} ) { let { datasourceId, tableName } = breakExternalTableId(tableId) - const table = await getExternalTable(appId, datasourceId, tableName) + const tables = await getAllExternalTables(appId, datasourceId) + const table = tables[tableName] if (!table) { throw `Unable to process query, table "${tableName}" not defined.` } // clean up row on ingress using schema filters = buildFilters(id, filters, table) + const relationships = buildRelationships(table) row = inputProcessing(row, table) if ( operation === DataSourceOperation.DELETE && @@ -116,6 +177,7 @@ async function handleRequest( filters, sort, paginate, + relationships, body: row, // pass an id filter into extra, purely for mysql/returning extra: { @@ -126,9 +188,9 @@ async function handleRequest( const response = await makeExternalQuery(appId, json) // we searched for rows in someway if (operation === DataSourceOperation.READ && Array.isArray(response)) { - return outputProcessing(response, table) + return outputProcessing(response, table, relationships, tables) } else { - row = outputProcessing(response, table)[0] + row = outputProcessing(response, table, relationships, tables)[0] return { row, table } } } @@ -270,7 +332,4 @@ exports.validate = async () => { return { valid: true } } -exports.fetchEnrichedRow = async () => { - // TODO: How does this work - throw "Not Implemented" -} +exports.fetchEnrichedRow = async () => {} diff --git a/packages/server/src/api/controllers/table/utils.js b/packages/server/src/api/controllers/table/utils.js index cdfd390027..78dae60ab1 100644 --- a/packages/server/src/api/controllers/table/utils.js +++ b/packages/server/src/api/controllers/table/utils.js @@ -204,15 +204,18 @@ class TableSaveFunctions { } } -exports.getExternalTable = async (appId, datasourceId, tableName) => { +exports.getAllExternalTables = async (appId, datasourceId) => { const db = new CouchDB(appId) const datasource = await db.get(datasourceId) if (!datasource || !datasource.entities) { throw "Datasource is not configured fully." } - return Object.values(datasource.entities).find( - entity => entity.name === tableName - ) + return datasource.entities +} + +exports.getExternalTable = async (appId, datasourceId, tableName) => { + const entities = await exports.getAllExternalTables(appId, datasourceId) + return entities[tableName] } exports.TableSaveFunctions = TableSaveFunctions diff --git a/packages/server/src/integrations/base/sql.js b/packages/server/src/integrations/base/sql.js index c7cc95fc3e..08eac09f0b 100644 --- a/packages/server/src/integrations/base/sql.js +++ b/packages/server/src/integrations/base/sql.js @@ -55,6 +55,25 @@ function addFilters(query, filters) { return query } +function addRelationships(query, fromTable, relationships) { + if (!relationships) { + return query + } + for (let relationship of relationships) { + const from = `${fromTable}.${relationship.from}` + const to = `${relationship.tableName}.${relationship.to}` + if (!relationship.through) { + query = query.innerJoin(relationship.tableName, from, to) + } else { + const through = relationship + query = query + .innerJoin(through.tableName, from, through.from) + .innerJoin(relationship.tableName, to, through.to) + } + } + return query +} + function buildCreate(knex, json, opts) { const { endpoint, body } = json let query = knex(endpoint.entityId) @@ -67,8 +86,9 @@ function buildCreate(knex, json, opts) { } function buildRead(knex, json, limit) { - let { endpoint, resource, filters, sort, paginate } = json - let query = knex(endpoint.entityId) + let { endpoint, resource, filters, sort, paginate, relationships } = json + const tableName = endpoint.entityId + let query = knex(tableName) // select all if not specified if (!resource) { resource = { fields: [] } @@ -81,6 +101,8 @@ function buildRead(knex, json, limit) { } // handle where query = addFilters(query, filters) + // handle join + query = addRelationships(query, tableName, relationships) // handle sorting if (sort) { for (let [key, value] of Object.entries(sort)) { diff --git a/packages/server/src/integrations/postgres.js b/packages/server/src/integrations/postgres.js index 6266e6ca64..f2d1f08162 100644 --- a/packages/server/src/integrations/postgres.js +++ b/packages/server/src/integrations/postgres.js @@ -153,6 +153,20 @@ class PostgresIntegration extends Sql { name: columnName, type: convertType(column.data_type, TYPE_MAP), } + + // // TODO: hack for testing + // if (tableName === "persons") { + // tables[tableName].primaryDisplay = "firstname" + // } + // if (columnName.toLowerCase() === "personid" && tableName === "tasks") { + // tables[tableName].schema[columnName] = { + // name: columnName, + // type: "link", + // tableId: buildExternalTableId(datasourceId, "persons"), + // relationshipType: "one-to-many", + // fieldName: "personid", + // } + // } } this.tables = tables } From 9c3869cfbf5d0875b6a66c3803473c8482892c9f Mon Sep 17 00:00:00 2001 From: Martin McKeaveney Date: Wed, 23 Jun 2021 22:07:18 +0100 Subject: [PATCH 03/62] adding relationships file --- .../CreateEditRelationship.svelte | 156 ++++++++++++++++++ .../CreateEditRelationship/TableSelect.svelte | 21 +++ 2 files changed, 177 insertions(+) create mode 100644 packages/builder/src/pages/builder/app/[application]/data/datasource/[selectedDatasource]/CreateEditRelationship/CreateEditRelationship.svelte create mode 100644 packages/builder/src/pages/builder/app/[application]/data/datasource/[selectedDatasource]/CreateEditRelationship/TableSelect.svelte diff --git a/packages/builder/src/pages/builder/app/[application]/data/datasource/[selectedDatasource]/CreateEditRelationship/CreateEditRelationship.svelte b/packages/builder/src/pages/builder/app/[application]/data/datasource/[selectedDatasource]/CreateEditRelationship/CreateEditRelationship.svelte new file mode 100644 index 0000000000..cbab467e40 --- /dev/null +++ b/packages/builder/src/pages/builder/app/[application]/data/datasource/[selectedDatasource]/CreateEditRelationship/CreateEditRelationship.svelte @@ -0,0 +1,156 @@ + + + + + +
+ + + {#each tables as table} + (relationship.from = table)}> + {table.name} + {#if relationship.from?._id === table._id} + + {/if} + + {/each} + + + + + {#each tables as table} + (relationship.to = table)}> + {table.name} + {#if relationship.to?._id === table._id} + + {/if} + + {/each} + + +
+ + {#if relationship.from && relationship.to} +
+ option.name} + getOptionValue={option => option.value} + /> +
+ {/if} + + {#if relationship?.relationshipType === RelationshipTypes.MANY_TO_MANY} + + + {#each tables as table} + (relationship.through = table)}> + {table.name} + {#if relationship.through?._id === table._id} + + {/if} + + {/each} + + + + {#if relationship.through} +
+ + + {#each Object.keys(relationship.through) as column} + (relationship.through.from = column)}> + {column} + {#if relationship.through.from?._id === column._id} + + {/if} + + {/each} + + + + + {#each Object.keys(relationship.through) as column} + (relationship.through.to = column)}> + {column} + {#if relationship.through.to?._id === column._id} + + {/if} + + {/each} + + +
+ {/if} + {/if} + + + \ No newline at end of file diff --git a/packages/builder/src/pages/builder/app/[application]/data/datasource/[selectedDatasource]/CreateEditRelationship/TableSelect.svelte b/packages/builder/src/pages/builder/app/[application]/data/datasource/[selectedDatasource]/CreateEditRelationship/TableSelect.svelte new file mode 100644 index 0000000000..8d4613e0b5 --- /dev/null +++ b/packages/builder/src/pages/builder/app/[application]/data/datasource/[selectedDatasource]/CreateEditRelationship/TableSelect.svelte @@ -0,0 +1,21 @@ + + + + + {#each tables as table} + select(table)}> + {table.name} + {#if selected} + + {/if} + + {/each} + + \ No newline at end of file From 81a4328544faf942990ac1f39c85c70b37344607 Mon Sep 17 00:00:00 2001 From: Martin McKeaveney Date: Fri, 25 Jun 2021 18:11:03 +0100 Subject: [PATCH 04/62] relationship configuration panel --- .../CreateEditRelationship.svelte | 79 ------------- .../CreateEditRelationship.svelte | 104 +++++++++--------- .../[selectedDatasource]/index.svelte | 2 +- 3 files changed, 53 insertions(+), 132 deletions(-) delete mode 100644 packages/builder/src/pages/builder/app/[application]/data/datasource/[selectedDatasource]/CreateEditRelationship.svelte diff --git a/packages/builder/src/pages/builder/app/[application]/data/datasource/[selectedDatasource]/CreateEditRelationship.svelte b/packages/builder/src/pages/builder/app/[application]/data/datasource/[selectedDatasource]/CreateEditRelationship.svelte deleted file mode 100644 index b9f6dbf092..0000000000 --- a/packages/builder/src/pages/builder/app/[application]/data/datasource/[selectedDatasource]/CreateEditRelationship.svelte +++ /dev/null @@ -1,79 +0,0 @@ - - - - {#if step === 0} - Select your table - {:else if step === 1} - - Step 2 - {:else if step === 2} - Step 3 - {/if} - - - \ No newline at end of file diff --git a/packages/builder/src/pages/builder/app/[application]/data/datasource/[selectedDatasource]/CreateEditRelationship/CreateEditRelationship.svelte b/packages/builder/src/pages/builder/app/[application]/data/datasource/[selectedDatasource]/CreateEditRelationship/CreateEditRelationship.svelte index cbab467e40..0ed5a93c0a 100644 --- a/packages/builder/src/pages/builder/app/[application]/data/datasource/[selectedDatasource]/CreateEditRelationship/CreateEditRelationship.svelte +++ b/packages/builder/src/pages/builder/app/[application]/data/datasource/[selectedDatasource]/CreateEditRelationship/CreateEditRelationship.svelte @@ -2,33 +2,44 @@ import { RelationshipTypes } from "constants/backend" import { Menu, MenuItem, MenuSection, Button, Input, Icon, ModalContent, RadioGroup, Heading } from "@budibase/bbui" + // "tasks_something": { + // "name": "tasks_something", + // "type": "link", + // "tableId": "whatever/othertable", + // "relationshipType": "one-to-many", + // }, + export let save export let datasource + export let from export let tables - - let relationship = {} + export let relationship = {} $: console.log(relationship) - $: console.log("ds", datasource) - $: valid = relationship.name && relationship.from && relationship.to && relationship.relationshipType + $: valid = relationship.name && relationship.tableId && relationship.relationshipType + $: from = tables.find(table => table._id === relationship.source) + $: to = tables.find(table => table._id === relationship.tableId) + $: through = tables.find(table => table._id === relationship.through) + $: linkTable = through || to - $: relationshipOptions = relationship.from && relationship.to ? [ + $: relationshipOptions = from && to ? [ { - name: `Many ${relationship.from.name} rows → many ${relationship.to.name} rows`, - alt: `Many ${relationship.from.name} rows → many ${relationship.to.name} rows`, + name: `Many ${from.name} rows → many ${to.name} rows`, + alt: `Many ${from.name} rows → many ${to.name} rows`, value: RelationshipTypes.MANY_TO_MANY, }, { - name: `One ${relationship.from.name} row → many ${relationship.to.name} rows`, - alt: `One ${relationship.from.name} rows → many ${relationship.to.name} rows`, + name: `One ${from.name} row → many ${to.name} rows`, + alt: `One ${from.name} rows → many ${to.name} rows`, value: RelationshipTypes.ONE_TO_MANY, - }, - { - name: `One ${relationship.from.name} row → many ${relationship.to.name} rows`, - alt: `One ${relationship.from.name} row → many ${relationship.to.name} rows`, - value: RelationshipTypes.MANY_TO_ONE, - }, + } ] : [] + + function onChangeRelationshipType(evt) { + if (evt.detail === RelationshipTypes.ONE_TO_MANY) { + relationship.through = null + } + } // save the relationship on to the datasource function saveRelationship() { @@ -67,9 +78,9 @@ {#each tables as table} - (relationship.from = table)}> + (relationship.source = table._id)}> {table.name} - {#if relationship.from?._id === table._id} + {#if relationship.source === table._id} {/if} @@ -79,9 +90,9 @@ {#each tables as table} - (relationship.to = table)}> + (relationship.tableId = table._id)}> {table.name} - {#if relationship.to?._id === table._id} + {#if relationship.tableId === table._id} {/if} @@ -90,61 +101,50 @@
- {#if relationship.from && relationship.to} + {#if from && to}
option.name} getOptionValue={option => option.value} />
- {/if} {#if relationship?.relationshipType === RelationshipTypes.MANY_TO_MANY} {#each tables as table} - (relationship.through = table)}> + (relationship.through = table._id)}> {table.name} - {#if relationship.through?._id === table._id} + {#if relationship.through === table._id} {/if} {/each} - - {#if relationship.through} -
- - - {#each Object.keys(relationship.through) as column} - (relationship.through.from = column)}> - {column} - {#if relationship.through.from?._id === column._id} - - {/if} - - {/each} - - - - - {#each Object.keys(relationship.through) as column} - (relationship.through.to = column)}> - {column} - {#if relationship.through.to?._id === column._id} - - {/if} - - {/each} - - -
- {/if} {/if} + + +
+ + + {#each Object.keys(linkTable.schema) as column} + (relationship.foreignKey = column)}> + {column} + {#if relationship.foreignKey === column} + + {/if} + + {/each} + + +
+ {/if} + \ No newline at end of file diff --git a/packages/server/.vscode/launch.json b/packages/server/.vscode/launch.json index 7417938376..d1774752cb 100644 --- a/packages/server/.vscode/launch.json +++ b/packages/server/.vscode/launch.json @@ -8,7 +8,19 @@ "type": "node", "request": "launch", "name": "Start Server", - "program": "${workspaceFolder}/src/index.js" + "sourceMaps": true, + "program": "${workspaceFolder}/src/index.ts", + "preLaunchTask": "npm: build", + "outFiles": ["${workspaceFolder}/dist/**/*.js"] + }, + { + "name": "TS", + "type": "node", + "request": "launch", + "runtimeExecutable": "node", + "runtimeArgs": ["--nolazy", "-r", "ts-node/register/transpile-only"], + "args": ["src/index.ts"], + "cwd": "${workspaceRoot}", }, { "type": "node", From 3cfbced94d16f0f57c6ed2906248083b1691fc79 Mon Sep 17 00:00:00 2001 From: mike12345567 Date: Thu, 1 Jul 2021 19:20:41 +0100 Subject: [PATCH 36/62] Re-writing external layer in typescript. --- .../api/controllers/row/ExternalRequest.ts | 424 ++++++++++++++++++ .../src/api/controllers/row/external.js | 108 +---- .../src/api/controllers/row/externalUtils.js | 259 ----------- packages/server/src/definitions/common.ts | 42 +- packages/server/src/definitions/datasource.ts | 25 +- packages/server/src/integrations/utils.ts | 3 - 6 files changed, 463 insertions(+), 398 deletions(-) create mode 100644 packages/server/src/api/controllers/row/ExternalRequest.ts delete mode 100644 packages/server/src/api/controllers/row/externalUtils.js diff --git a/packages/server/src/api/controllers/row/ExternalRequest.ts b/packages/server/src/api/controllers/row/ExternalRequest.ts new file mode 100644 index 0000000000..950cbef084 --- /dev/null +++ b/packages/server/src/api/controllers/row/ExternalRequest.ts @@ -0,0 +1,424 @@ +import { + Operation, + SearchFilters, + SortJson, + PaginationJson, + RelationshipsJson, +} from "../../../definitions/datasource" +import { Row, Table, FieldSchema } from "../../../definitions/common" +import { + breakRowIdField, + generateRowIdField, +} from "../../../integrations/utils" + +interface ManyRelationship { + tableId?: string + id?: string + isUpdate?: boolean + [key: string]: any +} + +interface RunConfig { + id: string + row: Row + filters: SearchFilters + sort: SortJson + paginate: PaginationJson +} + +module External { + const { makeExternalQuery } = require("./utils") + const { DataSourceOperation, FieldTypes } = require("../../../constants") + const { getAllExternalTables } = require("../table/utils") + const { breakExternalTableId } = require("../../../integrations/utils") + const { processObjectSync } = require("@budibase/string-templates") + const { cloneDeep } = require("lodash/fp") + + function buildFilters( + id: string | undefined, + filters: SearchFilters, + table: Table + ) { + const primary = table.primary + // if passed in array need to copy for shifting etc + let idCopy = cloneDeep(id) + if (filters) { + // need to map over the filters and make sure the _id field isn't present + for (let filter of Object.values(filters)) { + if (filter._id && primary) { + const parts = breakRowIdField(filter._id) + for (let field of primary) { + filter[field] = parts.shift() + } + } + // make sure this field doesn't exist on any filter + delete filter._id + } + } + // there is no id, just use the user provided filters + if (!idCopy || !table) { + return filters + } + // if used as URL parameter it will have been joined + if (!Array.isArray(idCopy)) { + idCopy = breakRowIdField(idCopy) + } + const equal: any = {} + if (primary && idCopy) { + for (let field of primary) { + // work through the ID and get the parts + equal[field] = idCopy.shift() + } + } + return { + equal, + } + } + + function generateIdForRow(row: Row, table: Table): string { + const primary = table.primary + if (!row || !primary) { + return "" + } + // build id array + let idParts = [] + for (let field of primary) { + if (row[field]) { + idParts.push(row[field]) + } + } + if (idParts.length === 0) { + return "" + } + return generateRowIdField(idParts) + } + + function basicProcessing(row: Row, table: Table) { + const thisRow: { [key: string]: any } = {} + // filter the row down to what is actually the row (not joined) + for (let fieldName of Object.keys(table.schema)) { + thisRow[fieldName] = row[fieldName] + } + thisRow._id = generateIdForRow(row, table) + thisRow.tableId = table._id + thisRow._rev = "rev" + return thisRow + } + + function isMany(field: FieldSchema) { + return ( + field.relationshipType && field.relationshipType.split("-")[0] === "many" + ) + } + + class ExternalRequest { + private appId: string + private operation: Operation + private tableId: string + private tables: { [key: string]: Table } + + constructor( + appId: string, + operation: Operation, + tableId: string, + tables: { [key: string]: Table } + ) { + this.appId = appId + this.operation = operation + this.tableId = tableId + this.tables = tables + } + + inputProcessing(row: Row, table: Table) { + if (!row) { + return { row, manyRelationships: [] } + } + // we don't really support composite keys for relationships, this is why [0] is used + // @ts-ignore + const tablePrimary: string = table.primary[0] + let newRow: Row = {}, + manyRelationships: ManyRelationship[] = [] + for (let [key, field] of Object.entries(table.schema)) { + // if set already, or not set just skip it + if (!row[key] || newRow[key]) { + continue + } + // if its not a link then just copy it over + if (field.type !== FieldTypes.LINK) { + newRow[key] = row[key] + continue + } + const { tableName: linkTableName } = breakExternalTableId(field.tableId) + // table has to exist for many to many + if (!this.tables[linkTableName]) { + continue + } + const linkTable = this.tables[linkTableName] + // @ts-ignore + const linkTablePrimary = linkTable.primary[0] + if (!isMany(field)) { + + newRow[field.foreignKey || linkTablePrimary] = breakRowIdField( + row[key][0] + )[0] + } else { + // we're not inserting a doc, will be a bunch of update calls + const isUpdate = !field.through + const thisKey: string = isUpdate ? "id" : linkTablePrimary + // @ts-ignore + const otherKey: string = isUpdate + ? field.foreignKey + : tablePrimary + row[key].map((relationship: any) => { + // we don't really support composite keys for relationships, this is why [0] is used + manyRelationships.push({ + tableId: field.through || field.tableId, + isUpdate, + [thisKey]: breakRowIdField(relationship)[0], + // leave the ID for enrichment later + [otherKey]: `{{ ${tablePrimary} }}`, + }) + }) + } + } + // we return the relationships that may need to be created in the through table + // we do this so that if the ID is generated by the DB it can be inserted + // after the fact + return { row: newRow, manyRelationships } + } + + updateRelationshipColumns( + row: Row, + rows: { [key: string]: Row }, + relationships: RelationshipsJson[] + ) { + const columns: { [key: string]: any } = {} + for (let relationship of relationships) { + const linkedTable = this.tables[relationship.tableName] + if (!linkedTable) { + continue + } + let linked = basicProcessing(row, linkedTable) + if (!linked._id) { + continue + } + // if not returning full docs then get the minimal links out + const display = linkedTable.primaryDisplay + linked = { + primaryDisplay: display ? linked[display] : undefined, + _id: linked._id, + } + columns[relationship.column] = linked + } + for (let [column, related] of Object.entries(columns)) { + if (!row._id) { + continue + } + const rowId: string = row._id + if (!Array.isArray(rows[rowId][column])) { + rows[rowId][column] = [] + } + // make sure relationship hasn't been found already + if ( + !rows[rowId][column].find( + (relation: Row) => relation._id === related._id + ) + ) { + rows[rowId][column].push(related) + } + } + return rows + } + + outputProcessing( + rows: Row[], + table: Table, + relationships: RelationshipsJson[] + ) { + if (rows[0].read === true) { + return [] + } + let finalRows: { [key: string]: Row } = {} + for (let row of rows) { + const rowId = generateIdForRow(row, table) + row._id = rowId + // this is a relationship of some sort + if (finalRows[rowId]) { + finalRows = this.updateRelationshipColumns( + row, + finalRows, + relationships + ) + continue + } + const thisRow = basicProcessing(row, table) + finalRows[thisRow._id] = thisRow + // do this at end once its been added to the final rows + finalRows = this.updateRelationshipColumns( + row, + finalRows, + relationships + ) + } + return Object.values(finalRows) + } + + buildRelationships(table: Table): RelationshipsJson[] { + const relationships = [] + for (let [fieldName, field] of Object.entries(table.schema)) { + if (field.type !== FieldTypes.LINK) { + continue + } + const { tableName: linkTableName } = breakExternalTableId(field.tableId) + // no table to link to, this is not a valid relationships + if (!this.tables[linkTableName]) { + continue + } + const linkTable = this.tables[linkTableName] + if (!table.primary || !linkTable.primary) { + continue + } + const definition = { + // if no foreign key specified then use the name of the field in other table + from: field.foreignKey || table.primary[0], + to: field.fieldName, + tableName: linkTableName, + through: undefined, + // need to specify where to put this back into + column: fieldName, + } + if (field.through) { + const { tableName: throughTableName } = breakExternalTableId( + field.through + ) + definition.through = throughTableName + // don't support composite keys for relationships + definition.from = table.primary[0] + definition.to = linkTable.primary[0] + } + relationships.push(definition) + } + return relationships + } + + async handleManyRelationships(row: Row, relationships: ManyRelationship[]) { + const { appId, tables } = this + const promises = [] + for (let relationship of relationships) { + const { tableId, isUpdate, id, ...rest } = relationship + const { datasourceId, tableName } = breakExternalTableId(tableId) + const linkedTable = tables[tableName] + if (!linkedTable) { + continue + } + const endpoint = { + datasourceId, + entityId: tableName, + operation: isUpdate + ? DataSourceOperation.UPDATE + : DataSourceOperation.CREATE, + } + promises.push( + makeExternalQuery(appId, { + endpoint, + // if we're doing many relationships then we're writing, only one response + body: processObjectSync(rest, row), + filters: buildFilters(id, {}, linkedTable), + }) + ) + } + await Promise.all(promises) + } + + /** + * This function is a bit crazy, but the exact purpose of it is to protect against the scenario in which + * you have column overlap in relationships, e.g. we join a few different tables and they all have the + * concept of an ID, but for some of them it will be null (if they say don't have a relationship). + * Creating the specific list of fields that we desire, and excluding the ones that are no use to us + * is more performant and has the added benefit of protecting against this scenario. + */ + buildFields(table: Table) { + function extractNonLinkFieldNames(table: Table, existing: string[] = []) { + return Object.entries(table.schema) + .filter( + column => + column[1].type !== FieldTypes.LINK && + !existing.find((field: string) => field.includes(column[0])) + ) + .map(column => `${table.name}.${column[0]}`) + } + let fields = extractNonLinkFieldNames(table) + for (let field of Object.values(table.schema)) { + if (field.type !== FieldTypes.LINK) { + continue + } + const { tableName: linkTableName } = breakExternalTableId(field.tableId) + const linkTable = this.tables[linkTableName] + if (linkTable) { + const linkedFields = extractNonLinkFieldNames(linkTable, fields) + fields = fields.concat(linkedFields) + } + } + return fields + } + + async run({ id, row, filters, sort, paginate }: RunConfig) { + const { appId, operation, tableId } = this + let { datasourceId, tableName } = breakExternalTableId(tableId) + if (!this.tables) { + this.tables = await getAllExternalTables(appId, datasourceId) + } + const table = this.tables[tableName] + if (!table) { + throw `Unable to process query, table "${tableName}" not defined.` + } + // clean up row on ingress using schema + filters = buildFilters(id, filters, table) + const relationships = this.buildRelationships(table) + const processed = this.inputProcessing(row, table) + row = processed.row + if ( + operation === DataSourceOperation.DELETE && + (filters == null || Object.keys(filters).length === 0) + ) { + throw "Deletion must be filtered" + } + let json = { + endpoint: { + datasourceId, + entityId: tableName, + operation, + }, + resource: { + // have to specify the fields to avoid column overlap + fields: this.buildFields(table), + }, + filters, + sort, + paginate, + relationships, + body: row, + // pass an id filter into extra, purely for mysql/returning + extra: { + idFilter: buildFilters(id || generateIdForRow(row, table), {}, table), + }, + } + // can't really use response right now + const response = await makeExternalQuery(appId, json) + // handle many to many relationships now if we know the ID (could be auto increment) + if (processed.manyRelationships) { + await this.handleManyRelationships( + response[0], + processed.manyRelationships + ) + } + const output = this.outputProcessing(response, table, relationships) + // if reading it'll just be an array of rows, return whole thing + return operation === DataSourceOperation.READ && Array.isArray(response) + ? output + : { row: output[0], table } + } + } + + module.exports = ExternalRequest +} diff --git a/packages/server/src/api/controllers/row/external.js b/packages/server/src/api/controllers/row/external.js index 7786dbcd45..96928fb2ae 100644 --- a/packages/server/src/api/controllers/row/external.js +++ b/packages/server/src/api/controllers/row/external.js @@ -1,4 +1,3 @@ -const { makeExternalQuery } = require("./utils") const { DataSourceOperation, SortDirection, @@ -9,112 +8,9 @@ const { breakExternalTableId, breakRowIdField, } = require("../../../integrations/utils") -const { - buildRelationships, - buildFilters, - inputProcessing, - outputProcessing, - generateIdForRow, - buildFields, -} = require("./externalUtils") -const { processObjectSync } = require("@budibase/string-templates") +const ExternalRequest = require("./ExternalRequest") -class ExternalRequest { - constructor(appId, operation, tableId, tables) { - this.appId = appId - this.operation = operation - this.tableId = tableId - this.tables = tables - } - - async handleManyRelationships(row, relationships) { - const { appId, tables } = this - const promises = [] - for (let relationship of relationships) { - const { tableId, isUpdate, id, ...rest } = relationship - const { datasourceId, tableName } = breakExternalTableId(tableId) - const linkedTable = tables[tableName] - if (!linkedTable) { - continue - } - const endpoint = { - datasourceId, - entityId: tableName, - operation: isUpdate ? DataSourceOperation.UPDATE : DataSourceOperation.CREATE, - } - promises.push( - makeExternalQuery(appId, { - endpoint, - // if we're doing many relationships then we're writing, only one response - body: processObjectSync(rest, row), - filters: buildFilters(id, {}, linkedTable) - }) - ) - } - await Promise.all(promises) - } - - async run({ id, row, filters, sort, paginate }) { - const { appId, operation, tableId } = this - let { datasourceId, tableName } = breakExternalTableId(tableId) - if (!this.tables) { - this.tables = await getAllExternalTables(appId, datasourceId) - } - const table = this.tables[tableName] - if (!table) { - throw `Unable to process query, table "${tableName}" not defined.` - } - // clean up row on ingress using schema - filters = buildFilters(id, filters, table) - const relationships = buildRelationships(table, this.tables) - const processed = inputProcessing(row, table, this.tables) - row = processed.row - if ( - operation === DataSourceOperation.DELETE && - (filters == null || Object.keys(filters).length === 0) - ) { - throw "Deletion must be filtered" - } - let json = { - endpoint: { - datasourceId, - entityId: tableName, - operation, - }, - resource: { - // have to specify the fields to avoid column overlap - fields: buildFields(table, this.tables), - }, - filters, - sort, - paginate, - relationships, - body: row, - // pass an id filter into extra, purely for mysql/returning - extra: { - idFilter: buildFilters(id || generateIdForRow(row, table), {}, table), - }, - } - // can't really use response right now - const response = await makeExternalQuery(appId, json) - // handle many to many relationships now if we know the ID (could be auto increment) - if (processed.manyRelationships) { - await this.handleManyRelationships(response[0], processed.manyRelationships) - } - const output = outputProcessing(response, table, relationships, this.tables) - // if reading it'll just be an array of rows, return whole thing - return operation === DataSourceOperation.READ && Array.isArray(response) - ? output - : { row: output[0], table } - } -} - -async function handleRequest( - appId, - operation, - tableId, - opts = {} -) { +async function handleRequest(appId, operation, tableId, opts = {}) { return new ExternalRequest(appId, operation, tableId, opts.tables).run(opts) } diff --git a/packages/server/src/api/controllers/row/externalUtils.js b/packages/server/src/api/controllers/row/externalUtils.js deleted file mode 100644 index 71f80ddb5a..0000000000 --- a/packages/server/src/api/controllers/row/externalUtils.js +++ /dev/null @@ -1,259 +0,0 @@ -const { - breakExternalTableId, - generateRowIdField, - breakRowIdField, -} = require("../../../integrations/utils") -const { FieldTypes } = require("../../../constants") -const { cloneDeep } = require("lodash/fp") - -function basicProcessing(row, table) { - const thisRow = {} - // filter the row down to what is actually the row (not joined) - for (let fieldName of Object.keys(table.schema)) { - thisRow[fieldName] = row[fieldName] - } - thisRow._id = exports.generateIdForRow(row, table) - thisRow.tableId = table._id - thisRow._rev = "rev" - return thisRow -} - -function isMany(field) { - return field.relationshipType.split("-")[0] === "many" -} - -exports.inputProcessing = (row, table, allTables) => { - if (!row) { - return { row, manyRelationships: [] } - } - let newRow = {}, - manyRelationships = [] - for (let [key, field] of Object.entries(table.schema)) { - // if set already, or not set just skip it - if (!row[key] || newRow[key]) { - continue - } - // if its not a link then just copy it over - if (field.type !== FieldTypes.LINK) { - newRow[key] = row[key] - continue - } - const { tableName: linkTableName } = breakExternalTableId(field.tableId) - // table has to exist for many to many - if (!allTables[linkTableName]) { - continue - } - const linkTable = allTables[linkTableName] - if (!isMany(field)) { - // we don't really support composite keys for relationships, this is why [0] is used - newRow[field.foreignKey || linkTable.primary] = breakRowIdField( - row[key][0] - )[0] - } else { - // we're not inserting a doc, will be a bunch of update calls - const isUpdate = !field.through - const thisKey = isUpdate ? "id" : linkTable.primary - const otherKey = isUpdate ? field.foreignKey : table.primary - row[key].map(relationship => { - // we don't really support composite keys for relationships, this is why [0] is used - manyRelationships.push({ - tableId: field.through || field.tableId, - isUpdate, - [thisKey]: breakRowIdField(relationship)[0], - // leave the ID for enrichment later - [otherKey]: `{{ ${table.primary} }}`, - }) - }) - } - } - // we return the relationships that may need to be created in the through table - // we do this so that if the ID is generated by the DB it can be inserted - // after the fact - return { row: newRow, manyRelationships } -} - -exports.generateIdForRow = (row, table) => { - if (!row) { - return null - } - const primary = table.primary - // build id array - let idParts = [] - for (let field of primary) { - if (row[field]) { - idParts.push(row[field]) - } - } - if (idParts.length === 0) { - return null - } - return generateRowIdField(idParts) -} - -exports.updateRelationshipColumns = (row, rows, relationships, allTables) => { - const columns = {} - for (let relationship of relationships) { - const linkedTable = allTables[relationship.tableName] - if (!linkedTable) { - continue - } - let linked = basicProcessing(row, linkedTable) - if (!linked._id) { - continue - } - // if not returning full docs then get the minimal links out - const display = linkedTable.primaryDisplay - linked = { - primaryDisplay: display ? linked[display] : undefined, - _id: linked._id, - } - columns[relationship.column] = linked - } - for (let [column, related] of Object.entries(columns)) { - if (!Array.isArray(rows[row._id][column])) { - rows[row._id][column] = [] - } - // make sure relationship hasn't been found already - if (!rows[row._id][column].find(relation => relation._id === related._id)) { - rows[row._id][column].push(related) - } - } - return rows -} - -exports.outputProcessing = (rows, table, relationships, allTables) => { - // if no rows this is what is returned? Might be PG only - if (rows[0].read === true) { - return [] - } - let finalRows = {} - for (let row of rows) { - row._id = exports.generateIdForRow(row, table) - // this is a relationship of some sort - if (finalRows[row._id]) { - finalRows = exports.updateRelationshipColumns( - row, - finalRows, - relationships, - allTables - ) - continue - } - const thisRow = basicProcessing(row, table) - finalRows[thisRow._id] = thisRow - // do this at end once its been added to the final rows - finalRows = exports.updateRelationshipColumns( - row, - finalRows, - relationships, - allTables - ) - } - return Object.values(finalRows) -} - -/** - * This function is a bit crazy, but the exact purpose of it is to protect against the scenario in which - * you have column overlap in relationships, e.g. we join a few different tables and they all have the - * concept of an ID, but for some of them it will be null (if they say don't have a relationship). - * Creating the specific list of fields that we desire, and excluding the ones that are no use to us - * is more performant and has the added benefit of protecting against this scenario. - * @param {Object} table The table we are retrieving fields for. - * @param {Object[]} allTables All of the tables that exist in the external data source, this is - * needed to work out what is needed from other tables based on relationships. - * @return {string[]} A list of fields like ["products.productid"] which can be used for an SQL select. - */ -exports.buildFields = (table, allTables) => { - function extractNonLinkFieldNames(table, existing = []) { - return Object.entries(table.schema) - .filter( - column => - column[1].type !== FieldTypes.LINK && - !existing.find(field => field.includes(column[0])) - ) - .map(column => `${table.name}.${column[0]}`) - } - let fields = extractNonLinkFieldNames(table) - for (let field of Object.values(table.schema)) { - if (field.type !== FieldTypes.LINK) { - continue - } - const { tableName: linkTableName } = breakExternalTableId(field.tableId) - const linkTable = allTables[linkTableName] - if (linkTable) { - const linkedFields = extractNonLinkFieldNames(linkTable, fields) - fields = fields.concat(linkedFields) - } - } - return fields -} - -exports.buildFilters = (id, filters, table) => { - const primary = table.primary - // if passed in array need to copy for shifting etc - let idCopy = cloneDeep(id) - if (filters) { - // need to map over the filters and make sure the _id field isn't present - for (let filter of Object.values(filters)) { - if (filter._id) { - const parts = breakRowIdField(filter._id) - for (let field of primary) { - filter[field] = parts.shift() - } - } - // make sure this field doesn't exist on any filter - delete filter._id - } - } - // there is no id, just use the user provided filters - if (!idCopy || !table) { - return filters - } - // if used as URL parameter it will have been joined - if (!Array.isArray(idCopy)) { - idCopy = breakRowIdField(idCopy) - } - const equal = {} - for (let field of primary) { - // work through the ID and get the parts - equal[field] = idCopy.shift() - } - return { - equal, - } -} - -exports.buildRelationships = (table, allTables) => { - const relationships = [] - for (let [fieldName, field] of Object.entries(table.schema)) { - if (field.type !== FieldTypes.LINK) { - continue - } - const { tableName: linkTableName } = breakExternalTableId(field.tableId) - // no table to link to, this is not a valid relationships - if (!allTables[linkTableName]) { - continue - } - const linkTable = allTables[linkTableName] - const definition = { - // if no foreign key specified then use the name of the field in other table - from: field.foreignKey || table.primary[0], - to: field.fieldName, - tableName: linkTableName, - through: undefined, - // need to specify where to put this back into - column: fieldName, - } - if (field.through) { - const { tableName: throughTableName } = breakExternalTableId( - field.through - ) - definition.through = throughTableName - // don't support composite keys for relationships - definition.from = table.primary[0] - definition.to = linkTable.primary[0] - } - relationships.push(definition) - } - return relationships -} diff --git a/packages/server/src/definitions/common.ts b/packages/server/src/definitions/common.ts index 26c12bd3c9..0115cc7685 100644 --- a/packages/server/src/definitions/common.ts +++ b/packages/server/src/definitions/common.ts @@ -3,29 +3,31 @@ interface Base { _rev?: string } -export interface TableSchema { - [key: string]: { - // TODO: replace with field types enum when done - type: string - fieldName?: string - name: string - tableId?: string - relationshipType?: string - through?: string - foreignKey?: string - constraints?: { - type?: string - email?: boolean - inclusion?: string[] - length?: { - minimum?: string | number - maximum?: string | number - } - presence?: boolean +export interface FieldSchema { + // TODO: replace with field types enum when done + type: string + fieldName?: string + name: string + tableId?: string + relationshipType?: string + through?: string + foreignKey?: string + constraints?: { + type?: string + email?: boolean + inclusion?: string[] + length?: { + minimum?: string | number + maximum?: string | number } + presence?: boolean } } +export interface TableSchema { + [key: string]: FieldSchema +} + export interface Table extends Base { type?: string views?: {} @@ -38,7 +40,7 @@ export interface Table extends Base { export interface Row extends Base { type?: string - tableId: string + tableId?: string [key: string]: any } diff --git a/packages/server/src/definitions/datasource.ts b/packages/server/src/definitions/datasource.ts index b7a3d47fff..ac52a32026 100644 --- a/packages/server/src/definitions/datasource.ts +++ b/packages/server/src/definitions/datasource.ts @@ -47,7 +47,7 @@ export interface Integration { } export interface SearchFilters { - allOr: boolean + allOr?: boolean string?: { [key: string]: string } @@ -77,11 +77,21 @@ export interface SearchFilters { } } +export interface SortJson { + [key: string]: SortDirection +} + +export interface PaginationJson { + limit: number + page: string | number +} + export interface RelationshipsJson { through?: string - from: string - to: string + from?: string + to?: string tableName: string + column: string } export interface QueryJson { @@ -94,13 +104,8 @@ export interface QueryJson { fields: string[] } filters?: SearchFilters - sort?: { - [key: string]: SortDirection - } - paginate?: { - limit: number - page: string | number - } + sort?: SortJson + paginate?: PaginationJson body?: object extra?: { idFilter?: SearchFilters diff --git a/packages/server/src/integrations/utils.ts b/packages/server/src/integrations/utils.ts index b668e6edeb..b8e9ee1dda 100644 --- a/packages/server/src/integrations/utils.ts +++ b/packages/server/src/integrations/utils.ts @@ -30,9 +30,6 @@ export function generateRowIdField(keyProps: any[] = []) { // should always return an array export function breakRowIdField(_id: string) { - if (!_id) { - return null - } const decoded = decodeURIComponent(_id) const parsed = JSON.parse(decoded) return Array.isArray(parsed) ? parsed : [parsed] From 052c1936cea50f15a229620ae27c8ea695f34ebf Mon Sep 17 00:00:00 2001 From: mike12345567 Date: Thu, 1 Jul 2021 19:23:29 +0100 Subject: [PATCH 37/62] Linting. --- packages/server/src/api/controllers/row/ExternalRequest.ts | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/packages/server/src/api/controllers/row/ExternalRequest.ts b/packages/server/src/api/controllers/row/ExternalRequest.ts index 950cbef084..fcb6c6b75c 100644 --- a/packages/server/src/api/controllers/row/ExternalRequest.ts +++ b/packages/server/src/api/controllers/row/ExternalRequest.ts @@ -157,7 +157,6 @@ module External { // @ts-ignore const linkTablePrimary = linkTable.primary[0] if (!isMany(field)) { - newRow[field.foreignKey || linkTablePrimary] = breakRowIdField( row[key][0] )[0] @@ -166,9 +165,7 @@ module External { const isUpdate = !field.through const thisKey: string = isUpdate ? "id" : linkTablePrimary // @ts-ignore - const otherKey: string = isUpdate - ? field.foreignKey - : tablePrimary + const otherKey: string = isUpdate ? field.foreignKey : tablePrimary row[key].map((relationship: any) => { // we don't really support composite keys for relationships, this is why [0] is used manyRelationships.push({ From 569af97664a60aceabd3ad8345c15301c8a2038a Mon Sep 17 00:00:00 2001 From: Martin McKeaveney Date: Thu, 1 Jul 2021 21:24:23 +0100 Subject: [PATCH 38/62] preventing relationship overrides --- .../CreateEditRelationship.svelte | 113 ++++++------------ .../[selectedDatasource]/index.svelte | 9 +- packages/server/.vscode/launch.json | 9 -- packages/server/src/integrations/postgres.ts | 19 ++- 4 files changed, 58 insertions(+), 92 deletions(-) diff --git a/packages/builder/src/pages/builder/app/[application]/data/datasource/[selectedDatasource]/CreateEditRelationship/CreateEditRelationship.svelte b/packages/builder/src/pages/builder/app/[application]/data/datasource/[selectedDatasource]/CreateEditRelationship/CreateEditRelationship.svelte index a30c08c135..902d8fa4e2 100644 --- a/packages/builder/src/pages/builder/app/[application]/data/datasource/[selectedDatasource]/CreateEditRelationship/CreateEditRelationship.svelte +++ b/packages/builder/src/pages/builder/app/[application]/data/datasource/[selectedDatasource]/CreateEditRelationship/CreateEditRelationship.svelte @@ -8,10 +8,10 @@ export let from export let plusTables export let relationship = {} + export let close let originalName = relationship.name - $: console.log(relationship) $: tableOptions = plusTables.map(table => ({ label: table.name, value: table._id })) $: valid = relationship.name && relationship.tableId && relationship.relationshipType $: from = plusTables.find(table => table._id === relationship.source) @@ -19,18 +19,6 @@ $: through = plusTables.find(table => table._id === relationship.through) $: linkTable = through || to - $: relationshipOptions = from && to ? [ - { - name: `Many ${from.name} rows → many ${to.name} rows`, - alt: `Many ${from.name} rows → many ${to.name} rows`, - value: RelationshipTypes.MANY_TO_MANY, - }, - { - name: `One ${from.name} row → many ${to.name} rows`, - alt: `One ${from.name} rows → many ${to.name} rows`, - value: RelationshipTypes.ONE_TO_MANY, - } - ] : [] $: relationshipTypes = [ { @@ -51,25 +39,44 @@ // save the relationship on to the datasource async function saveRelationship() { + const manyToMany = relationship.relationshipType === RelationshipTypes.MANY_TO_MANY // source of relationship datasource.entities[from.name].schema[relationship.name] = { type: "link", ...relationship } - // if (originalName !== from.name) { - // delete datasource.entities[from.name].schema[originalName] - // } - // save other side of relationship in the other schema datasource.entities[to.name].schema[relationship.name] = { + name: relationship.name, type: "link", - relationshipType: relationship.relationshipType === RelationshipTypes.MANY_TO_MANY ? RelationshipTypes.MANY_TO_MANY : RelationshipTypes.MANY_TO_ONE, - tableId: to._id + relationshipType: manyToMany ? RelationshipTypes.MANY_TO_MANY : RelationshipTypes.MANY_TO_ONE, + tableId: from._id, + fieldName: relationship.fieldName, + foreignKey: relationship.foreignKey } + // If relationship has been renamed + if (originalName !== relationship.name) { + delete datasource.entities[from.name].schema[originalName] + delete datasource.entities[to.name].schema[originalName] + } + + console.log({ + from: datasource.entities[from.name].schema[relationship.name], + to: datasource.entities[to.name].schema[relationship.name], + }) + await save() await tables.fetch() } + + async function deleteRelationship() { + delete datasource.entities[from.name].schema[relationship.name] + delete datasource.entities[to.name].schema[relationship.name] + await save() + await tables.fetch() + close() + } {/if} - {#if relationship?.relationshipType === RelationshipTypes.ONE_TO_MANY} + {#if relationship?.relationshipType === RelationshipTypes.ONE_TO_MANY && to} + {/each} + \ No newline at end of file diff --git a/packages/server/src/constants/definitions.ts b/packages/server/src/constants/definitions.ts index b39f2550ab..8ab995adc4 100644 --- a/packages/server/src/constants/definitions.ts +++ b/packages/server/src/constants/definitions.ts @@ -37,6 +37,6 @@ export interface BudibaseAppMetadata { name: string url: string instance: { _id: string } - updatedAt: Date, + updatedAt: Date createdAt: Date -} \ No newline at end of file +} diff --git a/packages/server/src/integrations/postgres.ts b/packages/server/src/integrations/postgres.ts index 547f6bf049..f4f4e20582 100644 --- a/packages/server/src/integrations/postgres.ts +++ b/packages/server/src/integrations/postgres.ts @@ -135,10 +135,7 @@ module PostgresModule { * Fetches the tables from the postgres table and assigns them to the datasource. * @param {*} datasourceId - datasourceId to fetch */ - async buildSchema( - datasourceId: string, - entities: Record - ) { + async buildSchema(datasourceId: string, entities: Record) { let tableKeys: { [key: string]: string[] } = {} try { const primaryKeysResponse = await this.client.query( @@ -173,7 +170,7 @@ module PostgresModule { // add the existing relationships from the entities if they exist, to prevent them from being overridden if (entities) { - const existingTableSchema = entities[tableName].schema + const existingTableSchema = entities[tableName].schema for (let key in existingTableSchema) { if (existingTableSchema[key].type === "link") { tables[tableName].schema[key] = existingTableSchema[key] From 56d83864ea2a834ba13793ea0b01da0c80c823de Mon Sep 17 00:00:00 2001 From: mike12345567 Date: Fri, 2 Jul 2021 14:35:05 +0100 Subject: [PATCH 41/62] Removing console logs and simplifying the relationship name. --- .../datasource/[selectedDatasource]/index.svelte | 12 ++++-------- 1 file changed, 4 insertions(+), 8 deletions(-) diff --git a/packages/builder/src/pages/builder/app/[application]/data/datasource/[selectedDatasource]/index.svelte b/packages/builder/src/pages/builder/app/[application]/data/datasource/[selectedDatasource]/index.svelte index 106a95f74c..756121b458 100644 --- a/packages/builder/src/pages/builder/app/[application]/data/datasource/[selectedDatasource]/index.svelte +++ b/packages/builder/src/pages/builder/app/[application]/data/datasource/[selectedDatasource]/index.svelte @@ -31,7 +31,6 @@ if (column.type !== "link") { continue } - console.log(`table - ${table.name} - ${column.name} - id: ${column._id} - ${column.main}`) // these relationships have an id to pair them to each other // one has a main for the from side const key = column.main ? "from" : "to" @@ -57,15 +56,12 @@ const fromTableName = getTableName(toCol.tableId) const toTableName = getTableName(fromCol.tableId) const throughTableName = getTableName(fromCol.through) - console.log(throughTableName) - let displayFrom = `${fromTableName} (${fromCol.name})` - let displayTo = `${toTableName} (${toCol.name})` let displayString if (throughTableName) { - displayString = `${displayFrom} through ${throughTableName} → ${displayTo}` + displayString = `${fromTableName} through ${throughTableName} → ${toTableName}` } else { - displayString = `${displayFrom} → ${displayTo}` + displayString = `${fromTableName} → ${toTableName}` } return displayString } @@ -219,13 +215,13 @@ on:click={() => openRelationshipModal(relationship.from, relationship.to)} > -

+

{buildRelationshipDisplayString( relationship.from, relationship.to )}

-

{relationship.from?.name} to {relationship.to?.name}

+

{relationship.from?.name} to {relationship.to?.name}

→

{/each} From 499c28d8839971be04f8001f3faf8680d2e97091 Mon Sep 17 00:00:00 2001 From: mike12345567 Date: Fri, 2 Jul 2021 14:36:24 +0100 Subject: [PATCH 42/62] Linting. --- .../CreateEditRelationship.svelte | 60 +++++++++++-------- .../[selectedDatasource]/index.svelte | 2 +- .../modals/EditDisplayColumnsModal.svelte | 23 ++++--- 3 files changed, 51 insertions(+), 34 deletions(-) diff --git a/packages/builder/src/pages/builder/app/[application]/data/datasource/[selectedDatasource]/CreateEditRelationship/CreateEditRelationship.svelte b/packages/builder/src/pages/builder/app/[application]/data/datasource/[selectedDatasource]/CreateEditRelationship/CreateEditRelationship.svelte index f15d7b22f1..7ff1225d1c 100644 --- a/packages/builder/src/pages/builder/app/[application]/data/datasource/[selectedDatasource]/CreateEditRelationship/CreateEditRelationship.svelte +++ b/packages/builder/src/pages/builder/app/[application]/data/datasource/[selectedDatasource]/CreateEditRelationship/CreateEditRelationship.svelte @@ -11,31 +11,40 @@ export let toRelationship = {} export let close - let originalFromName = fromRelationship.name, originalToName = toRelationship.name + let originalFromName = fromRelationship.name, + originalToName = toRelationship.name function isValid(relationship) { - if (relationship.relationshipType === RelationshipTypes.MANY_TO_MANY && !relationship.through) { + if ( + relationship.relationshipType === RelationshipTypes.MANY_TO_MANY && + !relationship.through + ) { return false } - return relationship.name && relationship.tableId && relationship.relationshipType + return ( + relationship.name && relationship.tableId && relationship.relationshipType + ) } - $: tableOptions = plusTables.map(table => ({ label: table.name, value: table._id })) + $: tableOptions = plusTables.map(table => ({ + label: table.name, + value: table._id, + })) $: fromTable = plusTables.find(table => table._id === toRelationship?.tableId) $: toTable = plusTables.find(table => table._id === fromRelationship?.tableId) $: through = plusTables.find(table => table._id === fromRelationship?.through) $: valid = toTable && fromTable && isValid(fromRelationship) $: linkTable = through || toTable $: relationshipTypes = [ - { - label: "Many", - value: RelationshipTypes.MANY_TO_MANY, - }, - { - label: "One", - value: RelationshipTypes.MANY_TO_ONE, - } - ] + { + label: "Many", + value: RelationshipTypes.MANY_TO_MANY, + }, + { + label: "One", + value: RelationshipTypes.MANY_TO_ONE, + }, + ] $: updateRelationshipType(fromRelationship?.relationshipType) function updateRelationshipType(fromType) { @@ -48,7 +57,8 @@ function buildRelationships() { // if any to many only need to check from - const manyToMany = fromRelationship.relationshipType === RelationshipTypes.MANY_TO_MANY + const manyToMany = + fromRelationship.relationshipType === RelationshipTypes.MANY_TO_MANY // main is simply used to know this is the side the user configured it from const id = uuid() let relateFrom = { @@ -97,9 +107,11 @@ async function saveRelationship() { buildRelationships() // source of relationship - datasource.entities[fromTable.name].schema[fromRelationship.name] = fromRelationship + datasource.entities[fromTable.name].schema[fromRelationship.name] = + fromRelationship // save other side of relationship in the other schema - datasource.entities[toTable.name].schema[toRelationship.name] = toRelationship + datasource.entities[toTable.name].schema[toRelationship.name] = + toRelationship // If relationship has been renamed if (originalFromName !== fromRelationship.name) { @@ -139,35 +151,35 @@
- - {:else if toTable} {/each} - \ No newline at end of file + From 922e209c72c14800a57b19cc0b369bfa03778e2d Mon Sep 17 00:00:00 2001 From: mike12345567 Date: Fri, 2 Jul 2021 14:49:47 +0100 Subject: [PATCH 43/62] Updating server test cases. --- .../server/src/api/routes/tests/datasource.spec.js | 2 +- packages/server/src/integrations/tests/sql.spec.js | 10 +++++----- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/packages/server/src/api/routes/tests/datasource.spec.js b/packages/server/src/api/routes/tests/datasource.spec.js index d53001b06e..a041de4310 100644 --- a/packages/server/src/api/routes/tests/datasource.spec.js +++ b/packages/server/src/api/routes/tests/datasource.spec.js @@ -94,7 +94,7 @@ describe("/datasources", () => { .expect(200) // this is mock data, can't test it expect(res.body).toBeDefined() - expect(pg.queryMock).toHaveBeenCalledWith(`select "name", "age" from "users" where "name" like $1 limit $2`, ["John%", 5000]) + expect(pg.queryMock).toHaveBeenCalledWith(`select "name", "age" from "users" where "users"."name" like $1 limit $2`, ["John%", 5000]) }) }) diff --git a/packages/server/src/integrations/tests/sql.spec.js b/packages/server/src/integrations/tests/sql.spec.js index 2b6badd92d..fb57fe79e7 100644 --- a/packages/server/src/integrations/tests/sql.spec.js +++ b/packages/server/src/integrations/tests/sql.spec.js @@ -81,7 +81,7 @@ describe("SQL query builder", () => { })) expect(query).toEqual({ bindings: ["John%", limit], - sql: `select * from "${TABLE_NAME}" where "name" like $1 limit $2` + sql: `select * from "${TABLE_NAME}" where "${TABLE_NAME}"."name" like $1 limit $2` }) }) @@ -98,7 +98,7 @@ describe("SQL query builder", () => { })) expect(query).toEqual({ bindings: [2, 10, limit], - sql: `select * from "${TABLE_NAME}" where "age" between $1 and $2 limit $3` + sql: `select * from "${TABLE_NAME}" where "${TABLE_NAME}"."age" between $1 and $2 limit $3` }) }) @@ -114,7 +114,7 @@ describe("SQL query builder", () => { })) expect(query).toEqual({ bindings: [10, "John", limit], - sql: `select * from "${TABLE_NAME}" where ("age" = $1) or ("name" = $2) limit $3` + sql: `select * from "${TABLE_NAME}" where ("${TABLE_NAME}"."age" = $1) or ("${TABLE_NAME}"."name" = $2) limit $3` }) }) @@ -139,7 +139,7 @@ describe("SQL query builder", () => { })) expect(query).toEqual({ bindings: ["John", 1001], - sql: `update "${TABLE_NAME}" set "name" = $1 where "id" = $2 returning *` + sql: `update "${TABLE_NAME}" set "name" = $1 where "${TABLE_NAME}"."id" = $2 returning *` }) }) @@ -151,7 +151,7 @@ describe("SQL query builder", () => { })) expect(query).toEqual({ bindings: [1001], - sql: `delete from "${TABLE_NAME}" where "id" = $1 returning *` + sql: `delete from "${TABLE_NAME}" where "${TABLE_NAME}"."id" = $1 returning *` }) }) From ec889320bcac1b69fd5009bbbe98bb3d94c5362b Mon Sep 17 00:00:00 2001 From: mike12345567 Date: Fri, 2 Jul 2021 18:42:01 +0100 Subject: [PATCH 44/62] Fixing issue with many to many through junction table not realising some exist, or some need deleted - as well as removing limit from details screen, this was blocking join statements and served no purpose (its already a search by equals). --- .../store/screenTemplates/rowDetailScreen.js | 1 - .../api/controllers/row/ExternalRequest.ts | 122 +++++++++++++++--- 2 files changed, 101 insertions(+), 22 deletions(-) diff --git a/packages/builder/src/builderStore/store/screenTemplates/rowDetailScreen.js b/packages/builder/src/builderStore/store/screenTemplates/rowDetailScreen.js index b41b4085a8..cb97bd53db 100644 --- a/packages/builder/src/builderStore/store/screenTemplates/rowDetailScreen.js +++ b/packages/builder/src/builderStore/store/screenTemplates/rowDetailScreen.js @@ -98,7 +98,6 @@ const createScreen = table => { valueType: "Binding", }, ], - limit: 1, paginate: false, }) diff --git a/packages/server/src/api/controllers/row/ExternalRequest.ts b/packages/server/src/api/controllers/row/ExternalRequest.ts index fcb6c6b75c..7e1ab52e83 100644 --- a/packages/server/src/api/controllers/row/ExternalRequest.ts +++ b/packages/server/src/api/controllers/row/ExternalRequest.ts @@ -33,6 +33,7 @@ module External { const { breakExternalTableId } = require("../../../integrations/utils") const { processObjectSync } = require("@budibase/string-templates") const { cloneDeep } = require("lodash/fp") + const { isEqual } = require("lodash") function buildFilters( id: string | undefined, @@ -93,6 +94,18 @@ module External { return generateRowIdField(idParts) } + function getEndpoint(tableId: string | undefined, operation: string) { + if (!tableId) { + return {} + } + const { datasourceId, tableName } = breakExternalTableId(tableId) + return { + datasourceId, + entityId: tableName, + operation, + } + } + function basicProcessing(row: Row, table: Table) { const thisRow: { [key: string]: any } = {} // filter the row down to what is actually the row (not joined) @@ -112,7 +125,7 @@ module External { } class ExternalRequest { - private appId: string + private readonly appId: string private operation: Operation private tableId: string private tables: { [key: string]: Table } @@ -173,7 +186,7 @@ module External { isUpdate, [thisKey]: breakRowIdField(relationship)[0], // leave the ID for enrichment later - [otherKey]: `{{ ${tablePrimary} }}`, + [otherKey]: `{{ literal ${tablePrimary} }}`, }) }) } @@ -184,6 +197,11 @@ module External { return { row: newRow, manyRelationships } } + /** + * This iterates through the returned rows and works out what elements of the rows + * actually match up to another row (based on primary keys) - this is pretty specific + * to SQL and the way that SQL relationships are returned based on joins. + */ updateRelationshipColumns( row: Row, rows: { [key: string]: Row }, @@ -260,6 +278,11 @@ module External { return Object.values(finalRows) } + /** + * Gets the list of relationship JSON structures based on the columns in the table, + * this will be used by the underlying library to build whatever relationship mechanism + * it has (e.g. SQL joins). + */ buildRelationships(table: Table): RelationshipsJson[] { const relationships = [] for (let [fieldName, field] of Object.entries(table.schema)) { @@ -298,31 +321,88 @@ module External { return relationships } + /** + * This is a cached lookup, of relationship records, this is mainly for creating/deleting junction + * information. + */ + async lookup(row: Row, relationship: ManyRelationship, cache: {[key: string]: Row[]} = {}) { + const { tableId, isUpdate, id, ...rest } = relationship + const { tableName } = breakExternalTableId(tableId) + const table = this.tables[tableName] + if (isUpdate) { + return { rows: [], table } + } + // if not updating need to make sure we have a list of all possible options + let fullKey: string = tableId + "/", rowKey: string = "" + for (let key of Object.keys(rest)) { + if (row[key]) { + fullKey += key + rowKey = key + } + } + if (cache[fullKey] == null) { + cache[fullKey] = await makeExternalQuery(this.appId, { + endpoint: getEndpoint(tableId, DataSourceOperation.READ), + filters: { + equal: { + [rowKey]: row[rowKey], + }, + }, + }) + } + return { rows: cache[fullKey], table } + } + + /** + * Once a row has been written we may need to update a many field, e.g. updating foreign keys + * in a bunch of rows in another table, or inserting/deleting rows from a junction table (many to many). + * This is quite a complex process and is handled by this function, there are a few things going on here: + * 1. If updating foreign keys its relatively simple, just create a filter for the row that needs updated + * and write the various components. + * 2. If junction table, then we lookup what exists already, write what doesn't exist, work out what + * isn't supposed to exist anymore and delete those. This is better than the usual method of delete them + * all and then re-create, as theres no chance of losing data (e.g. delete succeed, but write fail). + */ async handleManyRelationships(row: Row, relationships: ManyRelationship[]) { - const { appId, tables } = this + const { appId } = this + if (relationships.length === 0) { + return + } + // if we're creating (in a through table) need to wipe the existing ones first const promises = [] + const cache: { [key:string]: Row[] } = {} for (let relationship of relationships) { const { tableId, isUpdate, id, ...rest } = relationship - const { datasourceId, tableName } = breakExternalTableId(tableId) - const linkedTable = tables[tableName] - if (!linkedTable) { - continue + const body = processObjectSync(rest, row) + const { table, rows } = await this.lookup(row, relationship, cache) + const found = rows.find(row => isEqual(body, row)) + const operation = isUpdate ? DataSourceOperation.UPDATE : DataSourceOperation.CREATE + if (!found) { + promises.push( + makeExternalQuery(appId, { + endpoint: getEndpoint(tableId, operation), + // if we're doing many relationships then we're writing, only one response + body, + filters: buildFilters(id, {}, table), + }) + ) + } else { + // remove the relationship from the rows + rows.splice(rows.indexOf(found), 1) } - const endpoint = { - datasourceId, - entityId: tableName, - operation: isUpdate - ? DataSourceOperation.UPDATE - : DataSourceOperation.CREATE, + } + // finally if creating, cleanup any rows that aren't supposed to be here + for (let [key, rows] of Object.entries(cache)) { + // @ts-ignore + const tableId: string = key.split("/").shift() + const { tableName } = breakExternalTableId(tableId) + const table = this.tables[tableName] + for (let row of rows) { + promises.push(makeExternalQuery(this.appId, { + endpoint: getEndpoint(tableId, DataSourceOperation.DELETE), + filters: buildFilters(generateIdForRow(row, table), {}, table) + })) } - promises.push( - makeExternalQuery(appId, { - endpoint, - // if we're doing many relationships then we're writing, only one response - body: processObjectSync(rest, row), - filters: buildFilters(id, {}, linkedTable), - }) - ) } await Promise.all(promises) } From 3dc877f315abe47727b95978a284bfd096f6edab Mon Sep 17 00:00:00 2001 From: mike12345567 Date: Fri, 2 Jul 2021 18:44:01 +0100 Subject: [PATCH 45/62] Linting. --- .../api/controllers/row/ExternalRequest.ts | 25 +++++++++++++------ 1 file changed, 17 insertions(+), 8 deletions(-) diff --git a/packages/server/src/api/controllers/row/ExternalRequest.ts b/packages/server/src/api/controllers/row/ExternalRequest.ts index 7e1ab52e83..876ed18ff0 100644 --- a/packages/server/src/api/controllers/row/ExternalRequest.ts +++ b/packages/server/src/api/controllers/row/ExternalRequest.ts @@ -325,7 +325,11 @@ module External { * This is a cached lookup, of relationship records, this is mainly for creating/deleting junction * information. */ - async lookup(row: Row, relationship: ManyRelationship, cache: {[key: string]: Row[]} = {}) { + async lookup( + row: Row, + relationship: ManyRelationship, + cache: { [key: string]: Row[] } = {} + ) { const { tableId, isUpdate, id, ...rest } = relationship const { tableName } = breakExternalTableId(tableId) const table = this.tables[tableName] @@ -333,7 +337,8 @@ module External { return { rows: [], table } } // if not updating need to make sure we have a list of all possible options - let fullKey: string = tableId + "/", rowKey: string = "" + let fullKey: string = tableId + "/", + rowKey: string = "" for (let key of Object.keys(rest)) { if (row[key]) { fullKey += key @@ -370,13 +375,15 @@ module External { } // if we're creating (in a through table) need to wipe the existing ones first const promises = [] - const cache: { [key:string]: Row[] } = {} + const cache: { [key: string]: Row[] } = {} for (let relationship of relationships) { const { tableId, isUpdate, id, ...rest } = relationship const body = processObjectSync(rest, row) const { table, rows } = await this.lookup(row, relationship, cache) const found = rows.find(row => isEqual(body, row)) - const operation = isUpdate ? DataSourceOperation.UPDATE : DataSourceOperation.CREATE + const operation = isUpdate + ? DataSourceOperation.UPDATE + : DataSourceOperation.CREATE if (!found) { promises.push( makeExternalQuery(appId, { @@ -398,10 +405,12 @@ module External { const { tableName } = breakExternalTableId(tableId) const table = this.tables[tableName] for (let row of rows) { - promises.push(makeExternalQuery(this.appId, { - endpoint: getEndpoint(tableId, DataSourceOperation.DELETE), - filters: buildFilters(generateIdForRow(row, table), {}, table) - })) + promises.push( + makeExternalQuery(this.appId, { + endpoint: getEndpoint(tableId, DataSourceOperation.DELETE), + filters: buildFilters(generateIdForRow(row, table), {}, table), + }) + ) } } await Promise.all(promises) From 579bce8a6af60a875cab5a481dfcf84b0ab47e24 Mon Sep 17 00:00:00 2001 From: mike12345567 Date: Sat, 3 Jul 2021 10:26:37 +0100 Subject: [PATCH 46/62] Fixing little UI bugs. --- .../datasource/[selectedDatasource]/index.svelte | 12 +++++++++--- packages/server/src/integrations/postgres.ts | 6 +++++- 2 files changed, 14 insertions(+), 4 deletions(-) diff --git a/packages/builder/src/pages/builder/app/[application]/data/datasource/[selectedDatasource]/index.svelte b/packages/builder/src/pages/builder/app/[application]/data/datasource/[selectedDatasource]/index.svelte index 9905d80684..85f300119c 100644 --- a/packages/builder/src/pages/builder/app/[application]/data/datasource/[selectedDatasource]/index.svelte +++ b/packages/builder/src/pages/builder/app/[application]/data/datasource/[selectedDatasource]/index.svelte @@ -177,9 +177,11 @@ Update display columns {/if} - +
+ +
@@ -319,4 +321,8 @@ grid-gap: var(--spacing-l); grid-template-columns: 1fr 1fr; } + + .table-buttons div { + grid-column-end: -1; + } diff --git a/packages/server/src/integrations/postgres.ts b/packages/server/src/integrations/postgres.ts index f4f4e20582..935bfbeeea 100644 --- a/packages/server/src/integrations/postgres.ts +++ b/packages/server/src/integrations/postgres.ts @@ -134,6 +134,7 @@ module PostgresModule { /** * Fetches the tables from the postgres table and assigns them to the datasource. * @param {*} datasourceId - datasourceId to fetch + * @param entities - the tables that are to be built */ async buildSchema(datasourceId: string, entities: Record) { let tableKeys: { [key: string]: string[] } = {} @@ -169,9 +170,12 @@ module PostgresModule { } // add the existing relationships from the entities if they exist, to prevent them from being overridden - if (entities) { + if (entities && entities[tableName]) { const existingTableSchema = entities[tableName].schema for (let key in existingTableSchema) { + if (!existingTableSchema.hasOwnProperty(key)) { + continue + } if (existingTableSchema[key].type === "link") { tables[tableName].schema[key] = existingTableSchema[key] } From e170d9d146ffbb79ac52eb38a50108034d612f57 Mon Sep 17 00:00:00 2001 From: mike12345567 Date: Sat, 3 Jul 2021 11:15:01 +0100 Subject: [PATCH 47/62] Re-writing a bit so that it is aware some functionality is SQL only, makes future plus endpoints easier. --- .../api/controllers/row/ExternalRequest.ts | 30 ++++++++----- .../src/api/controllers/row/external.js | 12 +++-- packages/server/src/definitions/common.ts | 16 +++++++ packages/server/src/definitions/datasource.ts | 14 ++++++ packages/server/src/integrations/index.ts | 45 ++++++++++--------- packages/server/src/integrations/utils.ts | 10 +++++ 6 files changed, 92 insertions(+), 35 deletions(-) diff --git a/packages/server/src/api/controllers/row/ExternalRequest.ts b/packages/server/src/api/controllers/row/ExternalRequest.ts index 876ed18ff0..7e9f7d7077 100644 --- a/packages/server/src/api/controllers/row/ExternalRequest.ts +++ b/packages/server/src/api/controllers/row/ExternalRequest.ts @@ -5,7 +5,7 @@ import { PaginationJson, RelationshipsJson, } from "../../../definitions/datasource" -import { Row, Table, FieldSchema } from "../../../definitions/common" +import {Row, Table, FieldSchema, Datasource} from "../../../definitions/common" import { breakRowIdField, generateRowIdField, @@ -29,11 +29,11 @@ interface RunConfig { module External { const { makeExternalQuery } = require("./utils") const { DataSourceOperation, FieldTypes } = require("../../../constants") - const { getAllExternalTables } = require("../table/utils") - const { breakExternalTableId } = require("../../../integrations/utils") + const { breakExternalTableId, isSQL } = require("../../../integrations/utils") const { processObjectSync } = require("@budibase/string-templates") const { cloneDeep } = require("lodash/fp") const { isEqual } = require("lodash") + const CouchDB = require("../../../db") function buildFilters( id: string | undefined, @@ -128,18 +128,22 @@ module External { private readonly appId: string private operation: Operation private tableId: string - private tables: { [key: string]: Table } + private datasource: Datasource + private tables: { [key: string]: Table } = {} constructor( appId: string, operation: Operation, tableId: string, - tables: { [key: string]: Table } + datasource: Datasource ) { this.appId = appId this.operation = operation this.tableId = tableId - this.tables = tables + this.datasource = datasource + if (datasource && datasource.entities) { + this.tables = datasource.entities + } } inputProcessing(row: Row, table: Table) { @@ -451,10 +455,16 @@ module External { async run({ id, row, filters, sort, paginate }: RunConfig) { const { appId, operation, tableId } = this let { datasourceId, tableName } = breakExternalTableId(tableId) - if (!this.tables) { - this.tables = await getAllExternalTables(appId, datasourceId) + if (!this.datasource) { + const db = new CouchDB(appId) + this.datasource = await db.get(datasourceId) + if (!this.datasource || !this.datasource.entities) { + throw "No tables found, fetch tables before query." + } + this.tables = this.datasource.entities } const table = this.tables[tableName] + let isSql = isSQL(this.datasource) if (!table) { throw `Unable to process query, table "${tableName}" not defined.` } @@ -476,8 +486,8 @@ module External { operation, }, resource: { - // have to specify the fields to avoid column overlap - fields: this.buildFields(table), + // have to specify the fields to avoid column overlap (for SQL) + fields: isSql ? this.buildFields(table) : [], }, filters, sort, diff --git a/packages/server/src/api/controllers/row/external.js b/packages/server/src/api/controllers/row/external.js index 96928fb2ae..9f0a55e2cd 100644 --- a/packages/server/src/api/controllers/row/external.js +++ b/packages/server/src/api/controllers/row/external.js @@ -9,9 +9,10 @@ const { breakRowIdField, } = require("../../../integrations/utils") const ExternalRequest = require("./ExternalRequest") +const CouchDB = require("../../../db") async function handleRequest(appId, operation, tableId, opts = {}) { - return new ExternalRequest(appId, operation, tableId, opts.tables).run(opts) + return new ExternalRequest(appId, operation, tableId, opts.datasource).run(opts) } exports.patch = async ctx => { @@ -162,14 +163,19 @@ exports.fetchEnrichedRow = async ctx => { const id = ctx.params.rowId const tableId = ctx.params.tableId const { datasourceId, tableName } = breakExternalTableId(tableId) - const tables = await getAllExternalTables(appId, datasourceId) + const db = new CouchDB(appId) + const datasource = await db.get(datasourceId) + if (!datasource || !datasource.entities) { + ctx.throw(400, "Datasource has not been configured for plus API.") + } + const tables = datasource.entities const response = await handleRequest( appId, DataSourceOperation.READ, tableId, { id, - tables, + datasource, } ) const table = tables[tableName] diff --git a/packages/server/src/definitions/common.ts b/packages/server/src/definitions/common.ts index 0115cc7685..497f8f68f2 100644 --- a/packages/server/src/definitions/common.ts +++ b/packages/server/src/definitions/common.ts @@ -1,3 +1,5 @@ +import { SourceNames } from "./datasource" + interface Base { _id?: string _rev?: string @@ -82,3 +84,17 @@ export interface Automation extends Base { trigger?: AutomationStep } } + +export interface Datasource extends Base { + type: string + name: string + source: SourceNames + // the config is defined by the schema + config: { + [key: string]: string | number | boolean + } + plus: boolean + entities?: { + [key: string]: Table + } +} diff --git a/packages/server/src/definitions/datasource.ts b/packages/server/src/definitions/datasource.ts index ac52a32026..22f1998601 100644 --- a/packages/server/src/definitions/datasource.ts +++ b/packages/server/src/definitions/datasource.ts @@ -26,6 +26,20 @@ export enum DatasourceFieldTypes { JSON = "json", } +export enum SourceNames { + POSTGRES = "POSTGRES", + DYNAMODB = "DYNAMODB", + MONGODB = "MONGODB", + ELASTICSEARCH = "ELASTICSEARCH", + COUCHDB = "COUCHDB", + SQL_SERVER = "SQL_SERVER", + S3 = "S3", + AIRTABLE = "AIRTABLE", + MYSQL = "MYSQL", + ARANGODB = "ARANGODB", + REST = "REST", +} + export interface QueryDefinition { type: QueryTypes displayName?: string diff --git a/packages/server/src/integrations/index.ts b/packages/server/src/integrations/index.ts index 4999f0c867..c0acd6b225 100644 --- a/packages/server/src/integrations/index.ts +++ b/packages/server/src/integrations/index.ts @@ -9,33 +9,34 @@ const airtable = require("./airtable") const mysql = require("./mysql") const arangodb = require("./arangodb") const rest = require("./rest") +const { SourceNames } = require("../definitions/datasource") const DEFINITIONS = { - POSTGRES: postgres.schema, - DYNAMODB: dynamodb.schema, - MONGODB: mongodb.schema, - ELASTICSEARCH: elasticsearch.schema, - COUCHDB: couchdb.schema, - SQL_SERVER: sqlServer.schema, - S3: s3.schema, - AIRTABLE: airtable.schema, - MYSQL: mysql.schema, - ARANGODB: arangodb.schema, - REST: rest.schema, + [SourceNames.POSTGRES]: postgres.schema, + [SourceNames.DYNAMODB]: dynamodb.schema, + [SourceNames.MONGODB]: mongodb.schema, + [SourceNames.ELASTICSEARCH]: elasticsearch.schema, + [SourceNames.COUCHDB]: couchdb.schema, + [SourceNames.SQL_SERVER]: sqlServer.schema, + [SourceNames.S3]: s3.schema, + [SourceNames.AIRTABLE]: airtable.schema, + [SourceNames.MYSQL]: mysql.schema, + [SourceNames.ARANGODB]: arangodb.schema, + [SourceNames.REST]: rest.schema, } const INTEGRATIONS = { - POSTGRES: postgres.integration, - DYNAMODB: dynamodb.integration, - MONGODB: mongodb.integration, - ELASTICSEARCH: elasticsearch.integration, - COUCHDB: couchdb.integration, - S3: s3.integration, - SQL_SERVER: sqlServer.integration, - AIRTABLE: airtable.integration, - MYSQL: mysql.integration, - ARANGODB: arangodb.integration, - REST: rest.integration, + [SourceNames.POSTGRES]: postgres.integration, + [SourceNames.DYNAMODB]: dynamodb.integration, + [SourceNames.MONGODB]: mongodb.integration, + [SourceNames.ELASTICSEARCH]: elasticsearch.integration, + [SourceNames.COUCHDB]: couchdb.integration, + [SourceNames.SQL_SERVER]: s3.integration, + [SourceNames.S3]: sqlServer.integration, + [SourceNames.AIRTABLE]: airtable.integration, + [SourceNames.MYSQL]: mysql.integration, + [SourceNames.ARANGODB]: arangodb.integration, + [SourceNames.REST]: rest.integration, } module.exports = { diff --git a/packages/server/src/integrations/utils.ts b/packages/server/src/integrations/utils.ts index b8e9ee1dda..9140764094 100644 --- a/packages/server/src/integrations/utils.ts +++ b/packages/server/src/integrations/utils.ts @@ -1,4 +1,6 @@ import { SqlQuery } from "../definitions/datasource" +import { Datasource } from "../definitions/common" +import { SourceNames } from "../definitions/datasource" const { DocumentTypes, SEPARATOR } = require("../db/utils") const { FieldTypes } = require("../constants") @@ -51,3 +53,11 @@ export function getSqlQuery(query: SqlQuery | string): SqlQuery { return query } } + +export function isSQL(datasource: Datasource): boolean { + if (!datasource || !datasource.source) { + return false + } + const SQL = [SourceNames.POSTGRES, SourceNames.SQL_SERVER, SourceNames.MYSQL] + return SQL.indexOf(datasource.source) !== -1 +} From 2e18abd34293e33bc38c34c2b0785961bad1683a Mon Sep 17 00:00:00 2001 From: mike12345567 Date: Sat, 3 Jul 2021 11:16:29 +0100 Subject: [PATCH 48/62] Linting. --- packages/server/src/api/controllers/row/ExternalRequest.ts | 7 ++++++- packages/server/src/api/controllers/row/external.js | 5 +++-- 2 files changed, 9 insertions(+), 3 deletions(-) diff --git a/packages/server/src/api/controllers/row/ExternalRequest.ts b/packages/server/src/api/controllers/row/ExternalRequest.ts index 7e9f7d7077..855c64e4c1 100644 --- a/packages/server/src/api/controllers/row/ExternalRequest.ts +++ b/packages/server/src/api/controllers/row/ExternalRequest.ts @@ -5,7 +5,12 @@ import { PaginationJson, RelationshipsJson, } from "../../../definitions/datasource" -import {Row, Table, FieldSchema, Datasource} from "../../../definitions/common" +import { + Row, + Table, + FieldSchema, + Datasource, +} from "../../../definitions/common" import { breakRowIdField, generateRowIdField, diff --git a/packages/server/src/api/controllers/row/external.js b/packages/server/src/api/controllers/row/external.js index 9f0a55e2cd..3a96064a9f 100644 --- a/packages/server/src/api/controllers/row/external.js +++ b/packages/server/src/api/controllers/row/external.js @@ -3,7 +3,6 @@ const { SortDirection, FieldTypes, } = require("../../../constants") -const { getAllExternalTables } = require("../table/utils") const { breakExternalTableId, breakRowIdField, @@ -12,7 +11,9 @@ const ExternalRequest = require("./ExternalRequest") const CouchDB = require("../../../db") async function handleRequest(appId, operation, tableId, opts = {}) { - return new ExternalRequest(appId, operation, tableId, opts.datasource).run(opts) + return new ExternalRequest(appId, operation, tableId, opts.datasource).run( + opts + ) } exports.patch = async ctx => { From 1314edeef1e60a10822a22b3e01d9a0e6cea7f05 Mon Sep 17 00:00:00 2001 From: mike12345567 Date: Mon, 5 Jul 2021 09:29:16 +0100 Subject: [PATCH 49/62] Allowing user firstname and lastname to be updated through user portal. --- .../builder/portal/manage/users/[userId].svelte | 14 ++++++++++++-- 1 file changed, 12 insertions(+), 2 deletions(-) diff --git a/packages/builder/src/pages/builder/portal/manage/users/[userId].svelte b/packages/builder/src/pages/builder/portal/manage/users/[userId].svelte index ac5b569411..495e45e7c3 100644 --- a/packages/builder/src/pages/builder/portal/manage/users/[userId].svelte +++ b/packages/builder/src/pages/builder/portal/manage/users/[userId].svelte @@ -60,6 +60,16 @@ let toggleDisabled = false + async function updateUserFirstName(evt) { + await users.save({ ...$userFetch?.data, firstName: evt.target.value }) + await userFetch.refresh() + } + + async function updateUserLastName(evt) { + await users.save({ ...$userFetch?.data, lastName: evt.target.value }) + await userFetch.refresh() + } + async function toggleFlag(flagName, detail) { toggleDisabled = true await users.save({ ...$userFetch?.data, [flagName]: { global: detail } }) @@ -113,11 +123,11 @@
- +
- +
{#if userId !== $auth.user._id} From f3f2af1c1d5d848b19b2bb417dacf20723a6419f Mon Sep 17 00:00:00 2001 From: mike12345567 Date: Mon, 5 Jul 2021 09:38:01 +0100 Subject: [PATCH 50/62] Linting. --- .../builder/portal/manage/users/[userId].svelte | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/packages/builder/src/pages/builder/portal/manage/users/[userId].svelte b/packages/builder/src/pages/builder/portal/manage/users/[userId].svelte index 495e45e7c3..912506d0cf 100644 --- a/packages/builder/src/pages/builder/portal/manage/users/[userId].svelte +++ b/packages/builder/src/pages/builder/portal/manage/users/[userId].svelte @@ -123,11 +123,19 @@
- +
- +
{#if userId !== $auth.user._id} From dcd47746937332fb41127e591e5e9dde85cce254 Mon Sep 17 00:00:00 2001 From: mike12345567 Date: Mon, 5 Jul 2021 10:48:28 +0100 Subject: [PATCH 51/62] Re-working UI after some review. --- .../CreateEditRelationship.svelte | 78 +++++++------------ 1 file changed, 29 insertions(+), 49 deletions(-) diff --git a/packages/builder/src/pages/builder/app/[application]/data/datasource/[selectedDatasource]/CreateEditRelationship/CreateEditRelationship.svelte b/packages/builder/src/pages/builder/app/[application]/data/datasource/[selectedDatasource]/CreateEditRelationship/CreateEditRelationship.svelte index 7ff1225d1c..e1546dd0b1 100644 --- a/packages/builder/src/pages/builder/app/[application]/data/datasource/[selectedDatasource]/CreateEditRelationship/CreateEditRelationship.svelte +++ b/packages/builder/src/pages/builder/app/[application]/data/datasource/[selectedDatasource]/CreateEditRelationship/CreateEditRelationship.svelte @@ -1,6 +1,6 @@ -
-
- -
-
- -
+ - - + + {#if fromRelationship?.relationshipType === RelationshipTypes.MANY_TO_MANY} + + {:else if toTable} + +
{#if originalFromName !== null} @@ -192,19 +184,7 @@ From 8254880fa898bc4b8d0ae66af5c09e6412374666 Mon Sep 17 00:00:00 2001 From: mike12345567 Date: Mon, 5 Jul 2021 11:03:54 +0100 Subject: [PATCH 52/62] Quick fixes for UI, making sure relationships UI not displayed unless tables exist and fixing issue with through being sent up with one-to-many. --- .../CreateEditRelationship.svelte | 22 +++++++++++-------- .../[selectedDatasource]/index.svelte | 21 +++++++++--------- 2 files changed, 24 insertions(+), 19 deletions(-) diff --git a/packages/builder/src/pages/builder/app/[application]/data/datasource/[selectedDatasource]/CreateEditRelationship/CreateEditRelationship.svelte b/packages/builder/src/pages/builder/app/[application]/data/datasource/[selectedDatasource]/CreateEditRelationship/CreateEditRelationship.svelte index e1546dd0b1..ab5580cd88 100644 --- a/packages/builder/src/pages/builder/app/[application]/data/datasource/[selectedDatasource]/CreateEditRelationship/CreateEditRelationship.svelte +++ b/packages/builder/src/pages/builder/app/[application]/data/datasource/[selectedDatasource]/CreateEditRelationship/CreateEditRelationship.svelte @@ -61,6 +61,10 @@ fromRelationship.relationshipType === RelationshipTypes.MANY_TO_MANY // main is simply used to know this is the side the user configured it from const id = uuid() + if (!manyToMany) { + delete fromRelationship.through + delete toRelationship.through + } let relateFrom = { ...fromRelationship, type: "link", @@ -154,21 +158,21 @@ bind:value={toRelationship.tableId} /> {:else if toTable}
- {#if originalFromName !== null} + {#if originalFromName != null} {/if}
diff --git a/packages/builder/src/pages/builder/app/[application]/data/datasource/[selectedDatasource]/index.svelte b/packages/builder/src/pages/builder/app/[application]/data/datasource/[selectedDatasource]/index.svelte index 790a97c4d2..b0377d2f27 100644 --- a/packages/builder/src/pages/builder/app/[application]/data/datasource/[selectedDatasource]/index.svelte +++ b/packages/builder/src/pages/builder/app/[application]/data/datasource/[selectedDatasource]/index.svelte @@ -179,7 +179,7 @@ {/if}
@@ -203,12 +203,12 @@
Relationships Create relationship
- Tell budibase how your tables are related to get even more smart - features. + Tell budibase how your tables are related to get even more smart + features. {/if}
From b4d91769ccf0107a3240fe803a772884d98aa6c0 Mon Sep 17 00:00:00 2001 From: mike12345567 Date: Mon, 5 Jul 2021 15:14:45 +0100 Subject: [PATCH 54/62] Fixing issue with single quotes in strings breaking JSON parsing. --- packages/server/src/integrations/utils.ts | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/packages/server/src/integrations/utils.ts b/packages/server/src/integrations/utils.ts index 968d9da58e..36880ab913 100644 --- a/packages/server/src/integrations/utils.ts +++ b/packages/server/src/integrations/utils.ts @@ -25,6 +25,8 @@ export function generateRowIdField(keyProps: any[] = []) { keyProps = [keyProps] } // this conserves order and types + // we have to swap the double quotes to single quotes for use in HBS statements + // when using the literal helper the double quotes can break things return encodeURIComponent(JSON.stringify(keyProps).replace(/"/g, "'")) } @@ -33,7 +35,11 @@ export function breakRowIdField(_id: string) { if (!_id) { return null } - return JSON.parse(decodeURIComponent(_id)) + // have to replace on the way back as we swapped out the double quotes + // when encoding, but JSON can't handle the single quotes + const decoded = decodeURIComponent(_id).replace(/'/g, "\"") + const parsed = JSON.parse(decoded) + return Array.isArray(parsed) ? parsed : [parsed] } export function convertType(type: string, map: { [key: string]: any }) { From abc19bc2c1c5f90ac10bcefd2b6fc4286d49db09 Mon Sep 17 00:00:00 2001 From: mike12345567 Date: Mon, 5 Jul 2021 15:16:32 +0100 Subject: [PATCH 55/62] Linting. --- packages/server/src/integrations/utils.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/server/src/integrations/utils.ts b/packages/server/src/integrations/utils.ts index 36880ab913..87ed185ba3 100644 --- a/packages/server/src/integrations/utils.ts +++ b/packages/server/src/integrations/utils.ts @@ -37,7 +37,7 @@ export function breakRowIdField(_id: string) { } // have to replace on the way back as we swapped out the double quotes // when encoding, but JSON can't handle the single quotes - const decoded = decodeURIComponent(_id).replace(/'/g, "\"") + const decoded = decodeURIComponent(_id).replace(/'/g, '"') const parsed = JSON.parse(decoded) return Array.isArray(parsed) ? parsed : [parsed] } From 54dae1b5a2a68d7771552b3e7669f61e626c4055 Mon Sep 17 00:00:00 2001 From: Budibase Release Bot <> Date: Mon, 5 Jul 2021 14:38:21 +0000 Subject: [PATCH 56/62] v0.9.70 --- lerna.json | 2 +- packages/auth/package.json | 2 +- packages/bbui/package.json | 2 +- packages/builder/package.json | 8 ++++---- packages/cli/package.json | 2 +- packages/client/package.json | 8 ++++---- packages/server/package.json | 10 +++++----- packages/standard-components/package.json | 4 ++-- packages/string-templates/package.json | 2 +- packages/worker/package.json | 6 +++--- 10 files changed, 23 insertions(+), 23 deletions(-) diff --git a/lerna.json b/lerna.json index f5c82b9cb7..c35a2dbdc9 100644 --- a/lerna.json +++ b/lerna.json @@ -1,5 +1,5 @@ { - "version": "0.9.69", + "version": "0.9.70", "npmClient": "yarn", "packages": [ "packages/*" diff --git a/packages/auth/package.json b/packages/auth/package.json index 9581a4ce1f..650d41b633 100644 --- a/packages/auth/package.json +++ b/packages/auth/package.json @@ -1,6 +1,6 @@ { "name": "@budibase/auth", - "version": "0.9.69", + "version": "0.9.70", "description": "Authentication middlewares for budibase builder and apps", "main": "src/index.js", "author": "Budibase", diff --git a/packages/bbui/package.json b/packages/bbui/package.json index f7bb019803..9b2b2823ef 100644 --- a/packages/bbui/package.json +++ b/packages/bbui/package.json @@ -1,7 +1,7 @@ { "name": "@budibase/bbui", "description": "A UI solution used in the different Budibase projects.", - "version": "0.9.69", + "version": "0.9.70", "license": "AGPL-3.0", "svelte": "src/index.js", "module": "dist/bbui.es.js", diff --git a/packages/builder/package.json b/packages/builder/package.json index e2712c6bed..c13f883a3d 100644 --- a/packages/builder/package.json +++ b/packages/builder/package.json @@ -1,6 +1,6 @@ { "name": "@budibase/builder", - "version": "0.9.69", + "version": "0.9.70", "license": "AGPL-3.0", "private": true, "scripts": { @@ -65,10 +65,10 @@ } }, "dependencies": { - "@budibase/bbui": "^0.9.69", - "@budibase/client": "^0.9.69", + "@budibase/bbui": "^0.9.70", + "@budibase/client": "^0.9.70", "@budibase/colorpicker": "1.1.2", - "@budibase/string-templates": "^0.9.69", + "@budibase/string-templates": "^0.9.70", "@sentry/browser": "5.19.1", "@spectrum-css/page": "^3.0.1", "@spectrum-css/vars": "^3.0.1", diff --git a/packages/cli/package.json b/packages/cli/package.json index b80607af40..e4b1cba6c0 100644 --- a/packages/cli/package.json +++ b/packages/cli/package.json @@ -1,6 +1,6 @@ { "name": "@budibase/cli", - "version": "0.9.69", + "version": "0.9.70", "description": "Budibase CLI, for developers, self hosting and migrations.", "main": "src/index.js", "bin": { diff --git a/packages/client/package.json b/packages/client/package.json index 2ac54c9b7b..917c8c10bd 100644 --- a/packages/client/package.json +++ b/packages/client/package.json @@ -1,6 +1,6 @@ { "name": "@budibase/client", - "version": "0.9.69", + "version": "0.9.70", "license": "MPL-2.0", "module": "dist/budibase-client.js", "main": "dist/budibase-client.js", @@ -18,9 +18,9 @@ "dev:builder": "rollup -cw" }, "dependencies": { - "@budibase/bbui": "^0.9.69", - "@budibase/standard-components": "^0.9.69", - "@budibase/string-templates": "^0.9.69", + "@budibase/bbui": "^0.9.70", + "@budibase/standard-components": "^0.9.70", + "@budibase/string-templates": "^0.9.70", "regexparam": "^1.3.0", "shortid": "^2.2.15", "svelte-spa-router": "^3.0.5" diff --git a/packages/server/package.json b/packages/server/package.json index 95df6cca24..c080cf1f5b 100644 --- a/packages/server/package.json +++ b/packages/server/package.json @@ -1,7 +1,7 @@ { "name": "@budibase/server", "email": "hi@budibase.com", - "version": "0.9.69", + "version": "0.9.70", "description": "Budibase Web Server", "main": "src/index.js", "repository": { @@ -59,9 +59,9 @@ "author": "Budibase", "license": "AGPL-3.0-or-later", "dependencies": { - "@budibase/auth": "^0.9.69", - "@budibase/client": "^0.9.69", - "@budibase/string-templates": "^0.9.69", + "@budibase/auth": "^0.9.70", + "@budibase/client": "^0.9.70", + "@budibase/string-templates": "^0.9.70", "@elastic/elasticsearch": "7.10.0", "@koa/router": "8.0.0", "@sendgrid/mail": "7.1.1", @@ -114,7 +114,7 @@ "devDependencies": { "@babel/core": "^7.14.3", "@babel/preset-env": "^7.14.4", - "@budibase/standard-components": "^0.9.69", + "@budibase/standard-components": "^0.9.70", "@jest/test-sequencer": "^24.8.0", "@types/bull": "^3.15.1", "@types/jest": "^26.0.23", diff --git a/packages/standard-components/package.json b/packages/standard-components/package.json index 239525c765..5268004438 100644 --- a/packages/standard-components/package.json +++ b/packages/standard-components/package.json @@ -29,11 +29,11 @@ "keywords": [ "svelte" ], - "version": "0.9.69", + "version": "0.9.70", "license": "MIT", "gitHead": "d1836a898cab3f8ab80ee6d8f42be1a9eed7dcdc", "dependencies": { - "@budibase/bbui": "^0.9.69", + "@budibase/bbui": "^0.9.70", "@spectrum-css/link": "^3.1.3", "@spectrum-css/page": "^3.0.1", "@spectrum-css/vars": "^3.0.1", diff --git a/packages/string-templates/package.json b/packages/string-templates/package.json index fcfb2ff9b3..78f822028c 100644 --- a/packages/string-templates/package.json +++ b/packages/string-templates/package.json @@ -1,6 +1,6 @@ { "name": "@budibase/string-templates", - "version": "0.9.69", + "version": "0.9.70", "description": "Handlebars wrapper for Budibase templating.", "main": "src/index.cjs", "module": "dist/bundle.mjs", diff --git a/packages/worker/package.json b/packages/worker/package.json index e7f6cc5a59..d6ce2edce1 100644 --- a/packages/worker/package.json +++ b/packages/worker/package.json @@ -1,7 +1,7 @@ { "name": "@budibase/worker", "email": "hi@budibase.com", - "version": "0.9.69", + "version": "0.9.70", "description": "Budibase background service", "main": "src/index.js", "repository": { @@ -21,8 +21,8 @@ "author": "Budibase", "license": "AGPL-3.0-or-later", "dependencies": { - "@budibase/auth": "^0.9.69", - "@budibase/string-templates": "^0.9.69", + "@budibase/auth": "^0.9.70", + "@budibase/string-templates": "^0.9.70", "@koa/router": "^8.0.0", "aws-sdk": "^2.811.0", "bcryptjs": "^2.4.3", From c10e2da4f58e045ea2f07237598ffc02aae25f8d Mon Sep 17 00:00:00 2001 From: mike12345567 Date: Mon, 5 Jul 2021 17:33:36 +0100 Subject: [PATCH 57/62] Fixing issue with TS definitions. --- packages/server/src/integrations/utils.ts | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/packages/server/src/integrations/utils.ts b/packages/server/src/integrations/utils.ts index 9956dfb926..d0af0e99a9 100644 --- a/packages/server/src/integrations/utils.ts +++ b/packages/server/src/integrations/utils.ts @@ -33,13 +33,13 @@ export function generateRowIdField(keyProps: any[] = []) { } // should always return an array -export function breakRowIdField(_id: string) { +export function breakRowIdField(_id: string): any[] { if (!_id) { - return null + return [] } // have to replace on the way back as we swapped out the double quotes // when encoding, but JSON can't handle the single quotes - const decoded = decodeURIComponent(_id).replace(/'/g, '"') + const decoded: string = decodeURIComponent(_id).replace(/'/g, '"') const parsed = JSON.parse(decoded) return Array.isArray(parsed) ? parsed : [parsed] } From 65c436145abd45dc439928b37f6673f804b36742 Mon Sep 17 00:00:00 2001 From: Bernhard Hayden Date: Sun, 27 Jun 2021 16:46:04 +0200 Subject: [PATCH 58/62] Proof of concept OIDC implementation --- packages/auth/package.json | 1 + packages/auth/src/index.js | 3 +- packages/auth/src/middleware/index.js | 2 + packages/auth/src/middleware/passport/oidc.js | 111 ++++++++++++++++++ packages/auth/yarn.lock | 39 +++++- .../auth/_components/OIDCButton.svelte | 35 ++++++ .../src/pages/builder/auth/login.svelte | 2 + packages/worker/package.json | 1 + .../worker/src/api/controllers/admin/auth.js | 25 ++++ packages/worker/src/api/index.js | 8 ++ packages/worker/src/api/routes/admin/auth.js | 2 + packages/worker/src/index.js | 4 + packages/worker/yarn.lock | 39 +++++- 13 files changed, 263 insertions(+), 9 deletions(-) create mode 100644 packages/auth/src/middleware/passport/oidc.js create mode 100644 packages/builder/src/pages/builder/auth/_components/OIDCButton.svelte diff --git a/packages/auth/package.json b/packages/auth/package.json index 650d41b633..9155d37eeb 100644 --- a/packages/auth/package.json +++ b/packages/auth/package.json @@ -13,6 +13,7 @@ "koa-passport": "^4.1.4", "lodash": "^4.17.21", "node-fetch": "^2.6.1", + "@techpass/passport-openidconnect": "^0.3.0", "passport-google-auth": "^1.0.2", "passport-google-oauth": "^2.0.0", "passport-jwt": "^4.0.0", diff --git a/packages/auth/src/index.js b/packages/auth/src/index.js index 9582d6ffd6..cb4cb8d550 100644 --- a/packages/auth/src/index.js +++ b/packages/auth/src/index.js @@ -2,7 +2,7 @@ const passport = require("koa-passport") const LocalStrategy = require("passport-local").Strategy const JwtStrategy = require("passport-jwt").Strategy const { StaticDatabases } = require("./db/utils") -const { jwt, local, authenticated, google, auditLog } = require("./middleware") +const { jwt, local, authenticated, google, oidc, auditLog } = require("./middleware") const { setDB, getDB } = require("./db") // Strategies @@ -44,6 +44,7 @@ module.exports = { buildAuthMiddleware: authenticated, passport, google, + oidc, jwt: require("jsonwebtoken"), auditLog, }, diff --git a/packages/auth/src/middleware/index.js b/packages/auth/src/middleware/index.js index 2a249ce0f9..35c7d9c388 100644 --- a/packages/auth/src/middleware/index.js +++ b/packages/auth/src/middleware/index.js @@ -1,11 +1,13 @@ const jwt = require("./passport/jwt") const local = require("./passport/local") const google = require("./passport/google") +const oidc = require("./passport/oidc") const authenticated = require("./authenticated") const auditLog = require("./auditLog") module.exports = { google, + oidc, jwt, local, authenticated, diff --git a/packages/auth/src/middleware/passport/oidc.js b/packages/auth/src/middleware/passport/oidc.js new file mode 100644 index 0000000000..09c7e2a05e --- /dev/null +++ b/packages/auth/src/middleware/passport/oidc.js @@ -0,0 +1,111 @@ +const env = require("../../environment") +const jwt = require("jsonwebtoken") +const database = require("../../db") +const OIDCStrategy = require("@techpass/passport-openidconnect").Strategy +const { + StaticDatabases, + generateGlobalUserID, + ViewNames, +} = require("../../db/utils") + +// async function authenticate(token, tokenSecret, profile, done) { +async function authenticate(issuer, sub, profile, jwtClaims, accessToken, refreshToken, idToken, params, done) { + // Check the user exists in the instance DB by email + const db = database.getDB(StaticDatabases.GLOBAL.name) + + let dbUser + + const userId = generateGlobalUserID(profile.id) + + try { + // use the google profile id + dbUser = await db.get(userId) + } catch (err) { + const user = { + _id: userId, + provider: profile.provider, + roles: {}, + ...profile._json, + } + + // check if an account with the google email address exists locally + const users = await db.query(`database/${ViewNames.USER_BY_EMAIL}`, { + key: profile._json.email, + include_docs: true, + }) + + // Google user already exists by email + if (users.rows.length > 0) { + const existing = users.rows[0].doc + + // remove the local account to avoid conflicts + await db.remove(existing._id, existing._rev) + + // merge with existing account + user.roles = existing.roles + user.builder = existing.builder + user.admin = existing.admin + + const response = await db.post(user) + dbUser = user + dbUser._rev = response.rev + } else { + return done( + new Error( + "email does not yet exist. You must set up your local budibase account first." + ), + false + ) + } + } + + // authenticate + const payload = { + userId: dbUser._id, + builder: dbUser.builder, + email: dbUser.email, + } + + dbUser.token = jwt.sign(payload, env.JWT_SECRET, { + expiresIn: "1 day", + }) + + return done(null, dbUser) +} + +/** + * Create an instance of the google passport strategy. This wrapper fetches the configuration + * from couchDB rather than environment variables, using this factory is necessary for dynamically configuring passport. + * @returns Dynamically configured Passport Google Strategy + */ +exports.strategyFactory = async function () { + try { + + /* + const { clientID, clientSecret, callbackURL } = config + + if (!clientID || !clientSecret || !callbackURL) { + throw new Error( + "Configuration invalid. Must contain google clientID, clientSecret and callbackURL" + ) + } + */ + + return new OIDCStrategy( + { + issuer: "https://base.uri/auth/realms/realm_name", + authorizationURL: "https://base.uri/auth/realms/realm_name/protocol/openid-connect/auth", + tokenURL: "https://base.uri/auth/realms/realm_name/protocol/openid-connect/token", + userInfoURL: "https://base.uri/auth/realms/realm_name/protocol/openid-connect/userinfo", + clientID: "my_client_id", + clientSecret: "my_client_secret", + callbackURL: "http://localhost:10000/api/admin/auth/oidc/callback", + scope: "openid profile email", + }, + authenticate + ) + } catch (err) { + console.error(err) + throw new Error("Error constructing OIDC authentication strategy", err) + } +} diff --git a/packages/auth/yarn.lock b/packages/auth/yarn.lock index 80625a9345..d52ce0145c 100644 --- a/packages/auth/yarn.lock +++ b/packages/auth/yarn.lock @@ -2,6 +2,17 @@ # yarn lockfile v1 +"@techpass/passport-openidconnect@^0.3.0": + version "0.3.0" + resolved "https://registry.yarnpkg.com/@techpass/passport-openidconnect/-/passport-openidconnect-0.3.0.tgz#a60b2bbf3f262649a5a02d5d186219944acc3010" + integrity sha512-bVsPwl66s7J7GHxTPlW/RJYhZol9SshNznQsx83OOh9G+JWFGoeWxh+xbX+FTdJNoUvGIGbJnpWPY2wC6NOHPw== + dependencies: + base64url "^3.0.1" + oauth "^0.9.15" + passport-strategy "^1.0.0" + request "^2.88.0" + webfinger "^0.4.2" + ajv@^6.12.3: version "6.12.6" resolved "https://registry.yarnpkg.com/ajv/-/ajv-6.12.6.tgz#baf5a62e802b07d977034586f8c3baf5adf26df4" @@ -71,7 +82,7 @@ base64-js@^1.0.2, base64-js@^1.3.1: resolved "https://registry.yarnpkg.com/base64-js/-/base64-js-1.5.1.tgz#1b1b440160a5bf7ad40b650f095963481903930a" integrity sha512-AKpaYlHn8t4SVbOHCy+b5+KKgvR4vrsD8vbvrbiQJps7fKDTkjkDry6ji0rUJjC0kzbNePLwzxq8iypo41qeWA== -base64url@3.x.x: +base64url@3.x.x, base64url@^3.0.1: version "3.0.1" resolved "https://registry.yarnpkg.com/base64url/-/base64url-3.0.1.tgz#6399d572e2bc3f90a9a8b22d5dbb0a32d33f788d" integrity sha512-ir1UPr3dkwexU7FdV8qBBbNDRUhMmIekYMFZfi+C/sLNnRESKPl23nB9b2pltqfOQNnGzsDdId90AEtG5tCx4A== @@ -574,7 +585,7 @@ oauth-sign@~0.9.0: resolved "https://registry.yarnpkg.com/oauth-sign/-/oauth-sign-0.9.0.tgz#47a7b016baa68b5fa0ecf3dee08a85c679ac6455" integrity sha512-fexhUFFPTGV8ybAtSIGbV6gOkSv8UtRbDBnAyLQw4QPKkgNlsH2ByPGtMUqdWkos6YCRmAqViwgZrJc/mRDzZQ== -oauth@0.9.x: +oauth@0.9.x, oauth@^0.9.15: version "0.9.15" resolved "https://registry.yarnpkg.com/oauth/-/oauth-0.9.15.tgz#bd1fefaf686c96b75475aed5196412ff60cfb9c1" integrity sha1-vR/vr2hslrdUda7VGWQS/2DPucE= @@ -748,7 +759,7 @@ redis-parser@^3.0.0: dependencies: redis-errors "^1.0.0" -request@^2.72.0, request@^2.74.0: +request@^2.72.0, request@^2.74.0, request@^2.88.0: version "2.88.2" resolved "https://registry.yarnpkg.com/request/-/request-2.88.2.tgz#d73c918731cb5a87da047e207234146f664d12b3" integrity sha512-MsvtOrfG9ZcrOwAW+Qi+F6HbD0CWXEh9ou77uOb7FM2WPhwT7smM833PzanhJLsgXjN89Ir6V2PczXNnMpwKhw== @@ -794,7 +805,7 @@ sax@1.2.1: resolved "https://registry.yarnpkg.com/sax/-/sax-1.2.1.tgz#7b8e656190b228e81a66aea748480d828cd2d37a" integrity sha1-e45lYZCyKOgaZq6nSEgNgozS03o= -sax@>=0.6.0: +sax@>=0.1.1, sax@>=0.6.0: version "1.2.4" resolved "https://registry.yarnpkg.com/sax/-/sax-1.2.4.tgz#2816234e2378bddc4e5354fab5caa895df7100d9" integrity sha512-NqVDv9TpANUjFm0N8uM5GxL36UgKi9/atZw+x7YFnQ8ckwFGKrl4xX4yWtrey3UJm5nP1kUbnYgLopqWNSRhWw== @@ -829,6 +840,11 @@ standard-as-callback@^2.1.0: resolved "https://registry.yarnpkg.com/standard-as-callback/-/standard-as-callback-2.1.0.tgz#8953fc05359868a77b5b9739a665c5977bb7df45" integrity sha512-qoRRSyROncaz1z0mvYqIE4lCd9p2R90i6GxW3uZv5ucSu8tU7B5HXUP1gG8pVZsYNVaXjk8ClXHPttLyxAL48A== +step@0.0.x: + version "0.0.6" + resolved "https://registry.yarnpkg.com/step/-/step-0.0.6.tgz#143e7849a5d7d3f4a088fe29af94915216eeede2" + integrity sha1-FD54SaXX0/SgiP4pr5SRUhbu7eI= + string-template@~1.0.0: version "1.0.0" resolved "https://registry.yarnpkg.com/string-template/-/string-template-1.0.0.tgz#9e9f2233dc00f218718ec379a28a5673ecca8b96" @@ -943,11 +959,26 @@ verror@1.10.0: core-util-is "1.0.2" extsprintf "^1.2.0" +webfinger@^0.4.2: + version "0.4.2" + resolved "https://registry.yarnpkg.com/webfinger/-/webfinger-0.4.2.tgz#3477a6d97799461896039fcffc650b73468ee76d" + integrity sha1-NHem2XeZRhiWA5/P/GULc0aO520= + dependencies: + step "0.0.x" + xml2js "0.1.x" + wrappy@1: version "1.0.2" resolved "https://registry.yarnpkg.com/wrappy/-/wrappy-1.0.2.tgz#b5243d8f3ec1aa35f1364605bc0d1036e30ab69f" integrity sha1-tSQ9jz7BqjXxNkYFvA0QNuMKtp8= +xml2js@0.1.x: + version "0.1.14" + resolved "https://registry.yarnpkg.com/xml2js/-/xml2js-0.1.14.tgz#5274e67f5a64c5f92974cd85139e0332adc6b90c" + integrity sha1-UnTmf1pkxfkpdM2FE54DMq3GuQw= + dependencies: + sax ">=0.1.1" + xml2js@0.4.19: version "0.4.19" resolved "https://registry.yarnpkg.com/xml2js/-/xml2js-0.4.19.tgz#686c20f213209e94abf0d1bcf1efaa291c7827a7" diff --git a/packages/builder/src/pages/builder/auth/_components/OIDCButton.svelte b/packages/builder/src/pages/builder/auth/_components/OIDCButton.svelte new file mode 100644 index 0000000000..98f5d3efb9 --- /dev/null +++ b/packages/builder/src/pages/builder/auth/_components/OIDCButton.svelte @@ -0,0 +1,35 @@ + + +{#if show} + window.open("/api/admin/auth/oidc", "_blank")} + > +
+

Sign in with OIDC

+
+ +{/if} + + diff --git a/packages/builder/src/pages/builder/auth/login.svelte b/packages/builder/src/pages/builder/auth/login.svelte index 9fb984c73e..3850431a0f 100644 --- a/packages/builder/src/pages/builder/auth/login.svelte +++ b/packages/builder/src/pages/builder/auth/login.svelte @@ -12,6 +12,7 @@ import { goto, params } from "@roxi/routify" import { auth, organisation } from "stores/portal" import GoogleButton from "./_components/GoogleButton.svelte" + import OIDCButton from "./_components/OIDCButton.svelte" import Logo from "assets/bb-emblem.svg" import { onMount } from "svelte" @@ -61,6 +62,7 @@ Sign in to {company} + Sign in with email diff --git a/packages/worker/package.json b/packages/worker/package.json index d6ce2edce1..833f7b3ee9 100644 --- a/packages/worker/package.json +++ b/packages/worker/package.json @@ -39,6 +39,7 @@ "koa-static": "^5.0.0", "node-fetch": "^2.6.1", "nodemailer": "^6.5.0", + "@techpass/passport-openidconnect": "^0.3.0", "passport-google-oauth": "^2.0.0", "passport-jwt": "^4.0.0", "passport-local": "^1.0.0", diff --git a/packages/worker/src/api/controllers/admin/auth.js b/packages/worker/src/api/controllers/admin/auth.js index 5304ac85d1..374aa5c47d 100644 --- a/packages/worker/src/api/controllers/admin/auth.js +++ b/packages/worker/src/api/controllers/admin/auth.js @@ -1,5 +1,6 @@ const authPkg = require("@budibase/auth") const { google } = require("@budibase/auth/src/middleware") +const { oidc } = require("@budibase/auth/src/middleware") const { Configs, EmailTemplatePurpose } = require("../../../constants") const CouchDB = require("../../../db") const { sendEmail, isEmailConfigured } = require("../../../utilities/email") @@ -129,3 +130,27 @@ exports.googleAuth = async (ctx, next) => { } )(ctx, next) } + +// Minimal OIDC attempt + +exports.oidcPreAuth = async (ctx, next) => { + const strategy = await oidc.strategyFactory() + + return passport.authenticate(strategy, { + scope: ["profile", "email"], + })(ctx, next) +} + +exports.oidcAuth = async (ctx, next) => { + const strategy = await oidc.strategyFactory() + + return passport.authenticate( + strategy, + { successRedirect: "/", failureRedirect: "/error" }, + async (err, user) => { + authInternal(ctx, user, err) + + ctx.redirect("/") + } + )(ctx, next) +} diff --git a/packages/worker/src/api/index.js b/packages/worker/src/api/index.js index bda57863f6..c77c70089e 100644 --- a/packages/worker/src/api/index.js +++ b/packages/worker/src/api/index.js @@ -25,6 +25,14 @@ const PUBLIC_ENDPOINTS = [ route: "/api/admin/auth/google/callback", method: "GET", }, + { + route: "/api/admin/auth/oidc", + method: "GET", + }, + { + route: "/api/admin/auth/oidc/callback", + method: "GET", + }, { route: "/api/admin/auth/reset", method: "POST", diff --git a/packages/worker/src/api/routes/admin/auth.js b/packages/worker/src/api/routes/admin/auth.js index 04e30fc006..27f09f74f9 100644 --- a/packages/worker/src/api/routes/admin/auth.js +++ b/packages/worker/src/api/routes/admin/auth.js @@ -39,5 +39,7 @@ router .post("/api/admin/auth/logout", authController.logout) .get("/api/admin/auth/google", authController.googlePreAuth) .get("/api/admin/auth/google/callback", authController.googleAuth) + .get("/api/admin/auth/oidc", authController.oidcPreAuth) + .get("/api/admin/auth/oidc/callback", authController.oidcAuth) module.exports = router diff --git a/packages/worker/src/index.js b/packages/worker/src/index.js index f59f8bab15..4e105a1435 100644 --- a/packages/worker/src/index.js +++ b/packages/worker/src/index.js @@ -5,6 +5,7 @@ require("@budibase/auth").init(CouchDB) const Koa = require("koa") const destroyable = require("server-destroy") const koaBody = require("koa-body") +const koaSession = require("koa-session") const { passport } = require("@budibase/auth").auth const logger = require("koa-pino-logger") const http = require("http") @@ -13,8 +14,11 @@ const redis = require("./utilities/redis") const app = new Koa() +app.keys = ['secret', 'key']; + // set up top level koa middleware app.use(koaBody({ multipart: true })) +app.use(koaSession(app)) app.use( logger({ diff --git a/packages/worker/yarn.lock b/packages/worker/yarn.lock index 53f10856e8..1d4227363f 100644 --- a/packages/worker/yarn.lock +++ b/packages/worker/yarn.lock @@ -566,6 +566,17 @@ dependencies: defer-to-connect "^2.0.0" +"@techpass/passport-openidconnect@^0.3.0": + version "0.3.0" + resolved "https://registry.yarnpkg.com/@techpass/passport-openidconnect/-/passport-openidconnect-0.3.0.tgz#a60b2bbf3f262649a5a02d5d186219944acc3010" + integrity sha512-bVsPwl66s7J7GHxTPlW/RJYhZol9SshNznQsx83OOh9G+JWFGoeWxh+xbX+FTdJNoUvGIGbJnpWPY2wC6NOHPw== + dependencies: + base64url "^3.0.1" + oauth "^0.9.15" + passport-strategy "^1.0.0" + request "^2.88.0" + webfinger "^0.4.2" + "@types/babel__core@^7.0.0", "@types/babel__core@^7.1.7": version "7.1.14" resolved "https://registry.yarnpkg.com/@types/babel__core/-/babel__core-7.1.14.tgz#faaeefc4185ec71c389f4501ee5ec84b170cc402" @@ -1058,7 +1069,7 @@ base64-js@^1.0.2, base64-js@^1.3.1: resolved "https://registry.yarnpkg.com/base64-js/-/base64-js-1.5.1.tgz#1b1b440160a5bf7ad40b650f095963481903930a" integrity sha512-AKpaYlHn8t4SVbOHCy+b5+KKgvR4vrsD8vbvrbiQJps7fKDTkjkDry6ji0rUJjC0kzbNePLwzxq8iypo41qeWA== -base64url@3.x.x: +base64url@3.x.x, base64url@^3.0.1: version "3.0.1" resolved "https://registry.yarnpkg.com/base64url/-/base64url-3.0.1.tgz#6399d572e2bc3f90a9a8b22d5dbb0a32d33f788d" integrity sha512-ir1UPr3dkwexU7FdV8qBBbNDRUhMmIekYMFZfi+C/sLNnRESKPl23nB9b2pltqfOQNnGzsDdId90AEtG5tCx4A== @@ -4183,7 +4194,7 @@ oauth-sign@~0.9.0: resolved "https://registry.yarnpkg.com/oauth-sign/-/oauth-sign-0.9.0.tgz#47a7b016baa68b5fa0ecf3dee08a85c679ac6455" integrity sha512-fexhUFFPTGV8ybAtSIGbV6gOkSv8UtRbDBnAyLQw4QPKkgNlsH2ByPGtMUqdWkos6YCRmAqViwgZrJc/mRDzZQ== -oauth@0.9.x: +oauth@0.9.x, oauth@^0.9.15: version "0.9.15" resolved "https://registry.yarnpkg.com/oauth/-/oauth-0.9.15.tgz#bd1fefaf686c96b75475aed5196412ff60cfb9c1" integrity sha1-vR/vr2hslrdUda7VGWQS/2DPucE= @@ -4933,7 +4944,7 @@ request-promise-native@^1.0.9: stealthy-require "^1.1.1" tough-cookie "^2.3.3" -request@^2.88.2: +request@^2.88.0, request@^2.88.2: version "2.88.2" resolved "https://registry.yarnpkg.com/request/-/request-2.88.2.tgz#d73c918731cb5a87da047e207234146f664d12b3" integrity sha512-MsvtOrfG9ZcrOwAW+Qi+F6HbD0CWXEh9ou77uOb7FM2WPhwT7smM833PzanhJLsgXjN89Ir6V2PczXNnMpwKhw== @@ -5080,7 +5091,7 @@ sax@1.2.1: resolved "https://registry.yarnpkg.com/sax/-/sax-1.2.1.tgz#7b8e656190b228e81a66aea748480d828cd2d37a" integrity sha1-e45lYZCyKOgaZq6nSEgNgozS03o= -sax@>=0.6.0: +sax@>=0.1.1, sax@>=0.6.0: version "1.2.4" resolved "https://registry.yarnpkg.com/sax/-/sax-1.2.4.tgz#2816234e2378bddc4e5354fab5caa895df7100d9" integrity sha512-NqVDv9TpANUjFm0N8uM5GxL36UgKi9/atZw+x7YFnQ8ckwFGKrl4xX4yWtrey3UJm5nP1kUbnYgLopqWNSRhWw== @@ -5390,6 +5401,11 @@ stealthy-require@^1.1.1: resolved "https://registry.yarnpkg.com/stealthy-require/-/stealthy-require-1.1.1.tgz#35b09875b4ff49f26a777e509b3090a3226bf24b" integrity sha1-NbCYdbT/SfJqd35QmzCQoyJr8ks= +step@0.0.x: + version "0.0.6" + resolved "https://registry.yarnpkg.com/step/-/step-0.0.6.tgz#143e7849a5d7d3f4a088fe29af94915216eeede2" + integrity sha1-FD54SaXX0/SgiP4pr5SRUhbu7eI= + string-length@^4.0.1: version "4.0.2" resolved "https://registry.yarnpkg.com/string-length/-/string-length-4.0.2.tgz#a8a8dc7bd5c1a82b9b3c8b87e125f66871b6e57a" @@ -5923,6 +5939,14 @@ walker@^1.0.7, walker@~1.0.5: dependencies: makeerror "1.0.x" +webfinger@^0.4.2: + version "0.4.2" + resolved "https://registry.yarnpkg.com/webfinger/-/webfinger-0.4.2.tgz#3477a6d97799461896039fcffc650b73468ee76d" + integrity sha1-NHem2XeZRhiWA5/P/GULc0aO520= + dependencies: + step "0.0.x" + xml2js "0.1.x" + webidl-conversions@^5.0.0: version "5.0.0" resolved "https://registry.yarnpkg.com/webidl-conversions/-/webidl-conversions-5.0.0.tgz#ae59c8a00b121543a2acc65c0434f57b0fc11aff" @@ -6031,6 +6055,13 @@ xml-name-validator@^3.0.0: resolved "https://registry.yarnpkg.com/xml-name-validator/-/xml-name-validator-3.0.0.tgz#6ae73e06de4d8c6e47f9fb181f78d648ad457c6a" integrity sha512-A5CUptxDsvxKJEU3yO6DuWBSJz/qizqzJKOMIfUJHETbBw/sFaDxgd6fxm1ewUaM0jZ444Fc5vC5ROYurg/4Pw== +xml2js@0.1.x: + version "0.1.14" + resolved "https://registry.yarnpkg.com/xml2js/-/xml2js-0.1.14.tgz#5274e67f5a64c5f92974cd85139e0332adc6b90c" + integrity sha1-UnTmf1pkxfkpdM2FE54DMq3GuQw= + dependencies: + sax ">=0.1.1" + xml2js@0.4.19: version "0.4.19" resolved "https://registry.yarnpkg.com/xml2js/-/xml2js-0.4.19.tgz#686c20f213209e94abf0d1bcf1efaa291c7827a7" From 624dd20522521b15c334839485c2d9bd883b16af Mon Sep 17 00:00:00 2001 From: Rory Powell Date: Mon, 5 Jul 2021 17:16:45 +0100 Subject: [PATCH 59/62] Use configuration url to retrieve oidc endpoints The /.well-known/openid-configuration endpoint can be used to retrieve the majority of configuration needed for oidc Additionally refactor the callback url to be generated on the server side as this is a fixed endpoint. Add linting fixes --- packages/auth/src/index.js | 9 ++- packages/auth/src/middleware/passport/oidc.js | 68 ++++++++++++------- .../auth/_components/OIDCButton.svelte | 11 ++- .../worker/src/api/controllers/admin/auth.js | 13 +++- packages/worker/src/index.js | 2 +- 5 files changed, 65 insertions(+), 38 deletions(-) diff --git a/packages/auth/src/index.js b/packages/auth/src/index.js index cb4cb8d550..c56c5c5a05 100644 --- a/packages/auth/src/index.js +++ b/packages/auth/src/index.js @@ -2,7 +2,14 @@ const passport = require("koa-passport") const LocalStrategy = require("passport-local").Strategy const JwtStrategy = require("passport-jwt").Strategy const { StaticDatabases } = require("./db/utils") -const { jwt, local, authenticated, google, oidc, auditLog } = require("./middleware") +const { + jwt, + local, + authenticated, + google, + oidc, + auditLog, +} = require("./middleware") const { setDB, getDB } = require("./db") // Strategies diff --git a/packages/auth/src/middleware/passport/oidc.js b/packages/auth/src/middleware/passport/oidc.js index 09c7e2a05e..78a11784e4 100644 --- a/packages/auth/src/middleware/passport/oidc.js +++ b/packages/auth/src/middleware/passport/oidc.js @@ -1,6 +1,7 @@ const env = require("../../environment") const jwt = require("jsonwebtoken") const database = require("../../db") +const fetch = require("node-fetch") const OIDCStrategy = require("@techpass/passport-openidconnect").Strategy const { StaticDatabases, @@ -9,7 +10,17 @@ const { } = require("../../db/utils") // async function authenticate(token, tokenSecret, profile, done) { -async function authenticate(issuer, sub, profile, jwtClaims, accessToken, refreshToken, idToken, params, done) { +async function authenticate( + issuer, + sub, + profile, + jwtClaims, + accessToken, + refreshToken, + idToken, + params, + done +) { // Check the user exists in the instance DB by email const db = database.getDB(StaticDatabases.GLOBAL.name) @@ -18,7 +29,7 @@ async function authenticate(issuer, sub, profile, jwtClaims, accessToken, refres const userId = generateGlobalUserID(profile.id) try { - // use the google profile id + // use the OIDC profile id dbUser = await db.get(userId) } catch (err) { const user = { @@ -28,13 +39,13 @@ async function authenticate(issuer, sub, profile, jwtClaims, accessToken, refres ...profile._json, } - // check if an account with the google email address exists locally + // check if an account with the OIDC email address exists locally const users = await db.query(`database/${ViewNames.USER_BY_EMAIL}`, { key: profile._json.email, include_docs: true, }) - // Google user already exists by email + // OIDC user already exists by email if (users.rows.length > 0) { const existing = users.rows[0].doc @@ -74,36 +85,41 @@ async function authenticate(issuer, sub, profile, jwtClaims, accessToken, refres } /** - * Create an instance of the google passport strategy. This wrapper fetches the configuration + * Create an instance of the oidc passport strategy. This wrapper fetches the configuration * from couchDB rather than environment variables, using this factory is necessary for dynamically configuring passport. - * @returns Dynamically configured Passport Google Strategy + * @returns Dynamically configured Passport OIDC Strategy */ -exports.strategyFactory = async function () { +exports.strategyFactory = async function (callbackUrl) { try { + const configurationUrl = + "https://login.microsoftonline.com/2668c0dd-7ed2-4db3-b387-05b6f9204a70/v2.0/.well-known/openid-configuration" + const clientSecret = "g-ty~2iW.bo.88xj_QI6~hdc-H8mP2Xbnd" + const clientId = "bed2017b-2f53-42a9-8ef9-e58918935e07" - /* - const { clientID, clientSecret, callbackURL } = config - - if (!clientID || !clientSecret || !callbackURL) { + if (!clientId || !clientSecret || !callbackUrl || !configurationUrl) { throw new Error( - "Configuration invalid. Must contain google clientID, clientSecret and callbackURL" + "Configuration invalid. Must contain clientID, clientSecret, callbackUrl and configurationUrl" ) } - */ - return new OIDCStrategy( - { - issuer: "https://base.uri/auth/realms/realm_name", - authorizationURL: "https://base.uri/auth/realms/realm_name/protocol/openid-connect/auth", - tokenURL: "https://base.uri/auth/realms/realm_name/protocol/openid-connect/token", - userInfoURL: "https://base.uri/auth/realms/realm_name/protocol/openid-connect/userinfo", - clientID: "my_client_id", - clientSecret: "my_client_secret", - callbackURL: "http://localhost:10000/api/admin/auth/oidc/callback", - scope: "openid profile email", - }, - authenticate - ) + const response = await fetch(configurationUrl) + if (response.ok) { + const body = await response.json() + + return new OIDCStrategy( + { + issuer: body.issuer, + authorizationURL: body.authorization_endpoint, + tokenURL: body.token_endpoint, + userInfoURL: body.userinfo_endpoint, + clientID: clientId, + clientSecret: clientSecret, + callbackURL: callbackUrl, + scope: "profile email", + }, + authenticate + ) + } } catch (err) { console.error(err) throw new Error("Error constructing OIDC authentication strategy", err) diff --git a/packages/builder/src/pages/builder/auth/_components/OIDCButton.svelte b/packages/builder/src/pages/builder/auth/_components/OIDCButton.svelte index 98f5d3efb9..fbd67a6437 100644 --- a/packages/builder/src/pages/builder/auth/_components/OIDCButton.svelte +++ b/packages/builder/src/pages/builder/auth/_components/OIDCButton.svelte @@ -1,15 +1,12 @@ {#if show} - window.open("/api/admin/auth/oidc", "_blank")} - > + window.open("/api/admin/auth/oidc", "_blank")}>

Sign in with OIDC

@@ -25,10 +22,10 @@ padding-top: var(--spacing-xs); padding-bottom: var(--spacing-xs); } - .inner img { + /* .inner img { width: 18px; margin: 3px 10px 3px 3px; - } + } */ .inner p { margin: 0; } diff --git a/packages/worker/src/api/controllers/admin/auth.js b/packages/worker/src/api/controllers/admin/auth.js index 374aa5c47d..3c0f76c575 100644 --- a/packages/worker/src/api/controllers/admin/auth.js +++ b/packages/worker/src/api/controllers/admin/auth.js @@ -131,10 +131,17 @@ exports.googleAuth = async (ctx, next) => { )(ctx, next) } -// Minimal OIDC attempt +async function oidcStrategyFactory(ctx) { + const callbackUrl = `${ctx.protocol}://${ctx.host}/api/admin/auth/oidc/callback` + return oidc.strategyFactory(callbackUrl) +} +/** + * The initial call that OIDC authentication makes to take you to the configured OIDC login screen. + * On a successful login, you will be redirected to the oidcAuth callback route. + */ exports.oidcPreAuth = async (ctx, next) => { - const strategy = await oidc.strategyFactory() + const strategy = await oidcStrategyFactory(ctx) return passport.authenticate(strategy, { scope: ["profile", "email"], @@ -142,7 +149,7 @@ exports.oidcPreAuth = async (ctx, next) => { } exports.oidcAuth = async (ctx, next) => { - const strategy = await oidc.strategyFactory() + const strategy = await oidcStrategyFactory(ctx) return passport.authenticate( strategy, diff --git a/packages/worker/src/index.js b/packages/worker/src/index.js index 4e105a1435..8af1380552 100644 --- a/packages/worker/src/index.js +++ b/packages/worker/src/index.js @@ -14,7 +14,7 @@ const redis = require("./utilities/redis") const app = new Koa() -app.keys = ['secret', 'key']; +app.keys = ["secret", "key"] // set up top level koa middleware app.use(koaBody({ multipart: true })) From 0745eedbcce1287a7aa86a11ea4dbe93e900d5a9 Mon Sep 17 00:00:00 2001 From: Rory Powell Date: Mon, 5 Jul 2021 17:28:55 +0100 Subject: [PATCH 60/62] Improve error handling on openid-configuration request --- packages/auth/src/middleware/passport/oidc.js | 34 +++++++++++-------- 1 file changed, 19 insertions(+), 15 deletions(-) diff --git a/packages/auth/src/middleware/passport/oidc.js b/packages/auth/src/middleware/passport/oidc.js index 78a11784e4..d9a86ce574 100644 --- a/packages/auth/src/middleware/passport/oidc.js +++ b/packages/auth/src/middleware/passport/oidc.js @@ -103,23 +103,27 @@ exports.strategyFactory = async function (callbackUrl) { } const response = await fetch(configurationUrl) - if (response.ok) { - const body = await response.json() - return new OIDCStrategy( - { - issuer: body.issuer, - authorizationURL: body.authorization_endpoint, - tokenURL: body.token_endpoint, - userInfoURL: body.userinfo_endpoint, - clientID: clientId, - clientSecret: clientSecret, - callbackURL: callbackUrl, - scope: "profile email", - }, - authenticate - ) + if (!response.ok) { + throw new Error(`Unexpected response when fetching openid-configuration: ${response.statusText}`) } + + const body = await response.json() + + return new OIDCStrategy( + { + issuer: body.issuer, + authorizationURL: body.authorization_endpoint, + tokenURL: body.token_endpoint, + userInfoURL: body.userinfo_endpoint, + clientID: clientId, + clientSecret: clientSecret, + callbackURL: callbackUrl, + scope: "profile email", + }, + authenticate + ) + } catch (err) { console.error(err) throw new Error("Error constructing OIDC authentication strategy", err) From 42c8526ac2e882e70b3332c57cc695a76910a259 Mon Sep 17 00:00:00 2001 From: Rory Powell Date: Wed, 7 Jul 2021 13:28:55 +0100 Subject: [PATCH 61/62] Add info to 403 responses --- packages/worker/src/api/controllers/admin/auth.js | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/packages/worker/src/api/controllers/admin/auth.js b/packages/worker/src/api/controllers/admin/auth.js index 3c0f76c575..01717bffe0 100644 --- a/packages/worker/src/api/controllers/admin/auth.js +++ b/packages/worker/src/api/controllers/admin/auth.js @@ -11,16 +11,16 @@ const { checkResetPasswordCode } = require("../../../utilities/redis") const GLOBAL_DB = authPkg.StaticDatabases.GLOBAL.name -function authInternal(ctx, user, err = null) { +function authInternal(ctx, user, err = null, info = null) { if (err) { - return ctx.throw(403, "Unauthorized") + return ctx.throw(403, info? info : "Unauthorized") } const expires = new Date() expires.setDate(expires.getDate() + 1) if (!user) { - return ctx.throw(403, "Unauthorized") + return ctx.throw(403, info? info : "Unauthorized") } ctx.cookies.set(Cookies.Auth, user.token, { @@ -154,8 +154,8 @@ exports.oidcAuth = async (ctx, next) => { return passport.authenticate( strategy, { successRedirect: "/", failureRedirect: "/error" }, - async (err, user) => { - authInternal(ctx, user, err) + async (err, user, info) => { + authInternal(ctx, user, err, info) ctx.redirect("/") } From 1c39c2f0637f41bcf133e40ca564a48b8a988dd7 Mon Sep 17 00:00:00 2001 From: Rory Powell Date: Wed, 7 Jul 2021 13:45:33 +0100 Subject: [PATCH 62/62] Fallback to ID token to retrieve email when not available in passport profile (oidc userinfo) --- packages/auth/src/middleware/passport/oidc.js | 49 ++++++++++++++++--- 1 file changed, 41 insertions(+), 8 deletions(-) diff --git a/packages/auth/src/middleware/passport/oidc.js b/packages/auth/src/middleware/passport/oidc.js index d9a86ce574..f28aa030f9 100644 --- a/packages/auth/src/middleware/passport/oidc.js +++ b/packages/auth/src/middleware/passport/oidc.js @@ -9,7 +9,40 @@ const { ViewNames, } = require("../../db/utils") -// async function authenticate(token, tokenSecret, profile, done) { +/** + * Attempt to parse the users email address. + * + * It is not guaranteed that the email will be returned by the user info endpoint (e.g. github connected account used in azure ad). + * Fallback to the id token where possible. + * + * @param {*} profile The structured profile created by passport using the user info endpoint + * @param {*} jwtClaims The raw claims returned in the id token + */ +function getEmail(profile, jwtClaims) { + if (profile._json.email) { + return profile._json.email + } + + if (jwtClaims.email) { + return jwtClaims.email + } + + return null; +} + +/** + * + * @param {*} issuer The identity provider base URL + * @param {*} sub The user ID + * @param {*} profile The user profile information. Created by passport from the /userinfo response + * @param {*} jwtClaims The parsed id_token claims + * @param {*} accessToken The access_token for contacting the identity provider - may or may not be a JWT + * @param {*} refreshToken The refresh_token for obtaining a new access_token - usually not a JWT + * @param {*} idToken The id_token - always a JWT + * @param {*} params The response body from requesting an access_token + * @param {*} done The passport callback: err, user, info + * @returns + */ async function authenticate( issuer, sub, @@ -40,8 +73,13 @@ async function authenticate( } // check if an account with the OIDC email address exists locally + const email = getEmail(profile, jwtClaims) + if (!email) { + return done(null, false, { message: "No email address found" }) + } + const users = await db.query(`database/${ViewNames.USER_BY_EMAIL}`, { - key: profile._json.email, + key: email, include_docs: true, }) @@ -61,12 +99,7 @@ async function authenticate( dbUser = user dbUser._rev = response.rev } else { - return done( - new Error( - "email does not yet exist. You must set up your local budibase account first." - ), - false - ) + return done(null, false, { message: "Email does not yet exist. You must set up your local budibase account first." }) } }