diff --git a/packages/builder/src/pages/builder/app/[application]/data/datasource/[selectedDatasource]/index.svelte b/packages/builder/src/pages/builder/app/[application]/data/datasource/[selectedDatasource]/index.svelte
index 4560490b11..166c305d23 100644
--- a/packages/builder/src/pages/builder/app/[application]/data/datasource/[selectedDatasource]/index.svelte
+++ b/packages/builder/src/pages/builder/app/[application]/data/datasource/[selectedDatasource]/index.svelte
@@ -15,12 +15,22 @@
async function saveDatasource() {
try {
// Create datasource
- await datasources.save(datasource, { refresh: true })
+ await datasources.save(datasource)
notifications.success(`Datasource ${name} saved successfully.`)
unsaved = false
+ } catch (err) {
+ notifications.error(`Error saving datasource: ${err}`)
+ }
+ }
+
+ async function updateDatasourceSchema() {
+ try {
+ await datasources.updateSchema(datasource)
+ notifications.success(`Datasource ${name} schema saved successfully.`)
+ unsaved = false
await tables.fetch()
} catch (err) {
- notifications.error(`Error saving datasource: ${err}`)
+ notifications.error(`Error updating datasource schema: ${err}`)
}
}
@@ -71,7 +81,6 @@
on:change={setUnsaved}
/>
- {#if !integration.plus}
{/each}
+ {#if datasource.plus}
+
{/if}
diff --git a/packages/builder/src/stores/backend/datasources.js b/packages/builder/src/stores/backend/datasources.js
index 186068d28c..df669f4fd8 100644
--- a/packages/builder/src/stores/backend/datasources.js
+++ b/packages/builder/src/stores/backend/datasources.js
@@ -28,14 +28,34 @@ export function createDatasourcesStore() {
update(state => ({ ...state, selected: datasourceId }))
queries.update(state => ({ ...state, selected: null }))
},
- save: async (datasource, opts = {}) => {
- let url = "/api/datasources"
+ updateSchema: async (datasource) => {
+ let url = `/api/datasources/${datasource._id}/schema`
- if (datasource.plus && opts.refresh) {
- // Pull the latest tables from the datasource
- url += "?refresh=1"
+ const response = await api.post(url)
+ const json = await response.json()
+
+ if (response.status !== 200) {
+ throw new Error(json.message)
}
+ update(state => {
+ const currentIdx = state.list.findIndex(ds => ds._id === json._id)
+
+ const sources = state.list
+
+ if (currentIdx >= 0) {
+ sources.splice(currentIdx, 1, json)
+ } else {
+ sources.push(json)
+ }
+
+ return { list: sources, selected: json._id }
+ })
+ return json
+ },
+ save: async (datasource) => {
+ let url = "/api/datasources"
+
const response = await api.post(url, datasource)
const json = await response.json()
diff --git a/packages/server/src/api/controllers/datasource.js b/packages/server/src/api/controllers/datasource.js
index 44bcfcd9ee..b6d25b7b83 100644
--- a/packages/server/src/api/controllers/datasource.js
+++ b/packages/server/src/api/controllers/datasource.js
@@ -8,7 +8,6 @@ const {
getTableParams,
} = require("../../db/utils")
const { integrations } = require("../../integrations")
-const plusIntegrations = require("../../integrations/plus")
const { makeExternalQuery } = require("./row/utils")
exports.fetch = async function (ctx) {
@@ -40,6 +39,24 @@ exports.fetch = async function (ctx) {
ctx.body = [bbInternalDb, ...datasources]
}
+exports.buildSchemaFromDb = async function (ctx) {
+ const db = new CouchDB(ctx.appId)
+ const datasourceId = ctx.params.datasourceId
+ const datasource = await db.get(datasourceId)
+
+ const Connector = integrations[datasource.source]
+
+ // Connect to the DB and build the schema
+ const connector = new Connector(datasource.config)
+ await connector.buildSchema(datasource._id)
+ datasource.entities = connector.tables
+
+ const response = await db.post(datasource)
+ datasource._rev = response.rev
+
+ ctx.body = datasource
+}
+
exports.save = async function (ctx) {
const db = new CouchDB(ctx.appId)
const plus = ctx.request.body.plus
@@ -50,16 +67,6 @@ exports.save = async function (ctx) {
...ctx.request.body,
}
- // update the schema
- if (ctx.query.refresh) {
- const PlusConnector = plusIntegrations[datasource.source].integration
-
- const connector = new PlusConnector(ctx.request.body.config)
- await connector.init(datasource._id)
-
- datasource.entities = connector.tables
- }
-
const response = await db.post(datasource)
datasource._rev = response.rev
diff --git a/packages/server/src/api/controllers/table/index.js b/packages/server/src/api/controllers/table/index.js
index 35736fe399..db9ab4cd56 100644
--- a/packages/server/src/api/controllers/table/index.js
+++ b/packages/server/src/api/controllers/table/index.js
@@ -33,7 +33,7 @@ exports.fetch = async function (ctx) {
)
const external = externalTables.rows.flatMap(row => {
- return Object.values(row.doc.entities).map(entity => ({
+ return Object.values(row.doc.entities || {}).map(entity => ({
...entity,
sourceId: row.doc._id,
}))
diff --git a/packages/server/src/api/routes/datasource.js b/packages/server/src/api/routes/datasource.js
index 06964b61fd..e645cccd2b 100644
--- a/packages/server/src/api/routes/datasource.js
+++ b/packages/server/src/api/routes/datasource.js
@@ -72,6 +72,11 @@ router
generateQueryDatasourceSchema(),
datasourceController.query
)
+ .post(
+ "/api/datasources/:datasourceId/schema",
+ authorized(BUILDER),
+ datasourceController.buildSchemaFromDb
+ )
.post(
"/api/datasources",
authorized(BUILDER),
diff --git a/packages/server/src/integrations/index.js b/packages/server/src/integrations/index.js
index 6f90cee16c..bde154548e 100644
--- a/packages/server/src/integrations/index.js
+++ b/packages/server/src/integrations/index.js
@@ -24,7 +24,6 @@ const DEFINITIONS = {
MYSQL: mysql.schema,
ARANGODB: arangodb.schema,
REST: rest.schema,
- POSTGRES_PLUS: postgresPlus.schema,
}
const INTEGRATIONS = {
@@ -39,7 +38,6 @@ const INTEGRATIONS = {
MYSQL: mysql.integration,
ARANGODB: arangodb.integration,
REST: rest.integration,
- POSTGRES_PLUS: postgresPlus.integration,
}
module.exports = {
diff --git a/packages/server/src/integrations/plus/postgres.js b/packages/server/src/integrations/plus/postgres.js
index 2d5b747935..c7edc56f9f 100644
--- a/packages/server/src/integrations/plus/postgres.js
+++ b/packages/server/src/integrations/plus/postgres.js
@@ -1,135 +1,135 @@
-const Sql = require("../base/sql")
-const { Pool } = require("pg")
-const { FieldTypes } = require("../../constants")
-const { FIELD_TYPES } = require("../Integration")
-const { SEPARATOR } = require("@budibase/auth/db")
+// const Sql = require("../base/sql")
+// const { Pool } = require("pg")
+// const { FieldTypes } = require("../../constants")
+// const { FIELD_TYPES } = require("../Integration")
+// const { SEPARATOR } = require("@budibase/auth/db")
-const TYPE_MAP = {
- text: FieldTypes.LONGFORM,
- varchar: FieldTypes.STRING,
- integer: FieldTypes.NUMBER,
- bigint: FieldTypes.NUMBER,
- decimal: FieldTypes.NUMBER,
- smallint: FieldTypes.NUMBER,
- timestamp: FieldTypes.DATETIME,
- time: FieldTypes.DATETIME,
- boolean: FieldTypes.BOOLEAN,
- json: FIELD_TYPES.JSON,
-}
+// const TYPE_MAP = {
+// text: FieldTypes.LONGFORM,
+// varchar: FieldTypes.STRING,
+// integer: FieldTypes.NUMBER,
+// bigint: FieldTypes.NUMBER,
+// decimal: FieldTypes.NUMBER,
+// smallint: FieldTypes.NUMBER,
+// timestamp: FieldTypes.DATETIME,
+// time: FieldTypes.DATETIME,
+// boolean: FieldTypes.BOOLEAN,
+// json: FIELD_TYPES.JSON,
+// }
-const SCHEMA = {
- friendlyName: "PostgreSQL",
- description:
- "PostgreSQL, also known as Postgres, is a free and open-source relational database management system emphasizing extensibility and SQL compliance.",
- plus: true,
- datasource: {
- host: {
- type: FIELD_TYPES.STRING,
- default: "localhost",
- required: true,
- },
- port: {
- type: FIELD_TYPES.NUMBER,
- required: true,
- default: 5432,
- },
- database: {
- type: FIELD_TYPES.STRING,
- default: "postgres",
- required: true,
- },
- user: {
- type: FIELD_TYPES.STRING,
- default: "root",
- required: true,
- },
- password: {
- type: FIELD_TYPES.PASSWORD,
- default: "root",
- required: true,
- },
- ssl: {
- type: FIELD_TYPES.BOOLEAN,
- default: false,
- required: false,
- },
- },
-}
+// const SCHEMA = {
+// friendlyName: "PostgreSQL",
+// description:
+// "PostgreSQL, also known as Postgres, is a free and open-source relational database management system emphasizing extensibility and SQL compliance.",
+// plus: true,
+// datasource: {
+// host: {
+// type: FIELD_TYPES.STRING,
+// default: "localhost",
+// required: true,
+// },
+// port: {
+// type: FIELD_TYPES.NUMBER,
+// required: true,
+// default: 5432,
+// },
+// database: {
+// type: FIELD_TYPES.STRING,
+// default: "postgres",
+// required: true,
+// },
+// user: {
+// type: FIELD_TYPES.STRING,
+// default: "root",
+// required: true,
+// },
+// password: {
+// type: FIELD_TYPES.PASSWORD,
+// default: "root",
+// required: true,
+// },
+// ssl: {
+// type: FIELD_TYPES.BOOLEAN,
+// default: false,
+// required: false,
+// },
+// },
+// }
-class PostgresPlus extends Sql {
- static pool
- COLUMNS_SQL =
- "select * from information_schema.columns where table_schema = 'public'"
+// class PostgresPlus extends Sql {
+// static pool
+// COLUMNS_SQL =
+// "select * from information_schema.columns where table_schema = 'public'"
- PRIMARY_KEYS_SQL = `
- select tc.table_schema, tc.table_name, kc.column_name as primary_key
- from information_schema.table_constraints tc
- join
- information_schema.key_column_usage kc on kc.table_name = tc.table_name
- and kc.table_schema = tc.table_schema
- and kc.constraint_name = tc.constraint_name
- where tc.constraint_type = 'PRIMARY KEY';
- `
+// PRIMARY_KEYS_SQL = `
+// select tc.table_schema, tc.table_name, kc.column_name as primary_key
+// from information_schema.table_constraints tc
+// join
+// information_schema.key_column_usage kc on kc.table_name = tc.table_name
+// and kc.table_schema = tc.table_schema
+// and kc.constraint_name = tc.constraint_name
+// where tc.constraint_type = 'PRIMARY KEY';
+// `
- constructor(config, datasource) {
- super("pg")
- this.config = config
- this.datasource = datasource
+// constructor(config, datasource) {
+// super("pg")
+// this.config = config
+// this.datasource = datasource
- if (!this.pool) {
- this.pool = new Pool(this.config)
- }
+// if (!this.pool) {
+// this.pool = new Pool(this.config)
+// }
- this.client = this.pool
- }
+// this.client = this.pool
+// }
- async init(datasourceId) {
- let keys = []
- try {
- const primaryKeysResponse = await this.client.query(this.PRIMARY_KEYS_SQL)
- for (let table of primaryKeysResponse.rows) {
- keys.push(table.column_name || table.primary_key)
- }
- } catch (err) {
- // TODO: this try catch method isn't right
- keys = ["id"]
- }
+// async init(datasourceId) {
+// let keys = []
+// try {
+// const primaryKeysResponse = await this.client.query(this.PRIMARY_KEYS_SQL)
+// for (let table of primaryKeysResponse.rows) {
+// keys.push(table.column_name || table.primary_key)
+// }
+// } catch (err) {
+// // TODO: this try catch method isn't right
+// keys = ["id"]
+// }
- const columnsResponse = await this.client.query(this.COLUMNS_SQL)
- const tables = {}
+// const columnsResponse = await this.client.query(this.COLUMNS_SQL)
+// const tables = {}
- for (let column of columnsResponse.rows) {
- const tableName = column.table_name
- const columnName = column.column_name
+// for (let column of columnsResponse.rows) {
+// const tableName = column.table_name
+// const columnName = column.column_name
- // table key doesn't exist yet
- if (!tables[tableName]) {
- tables[tableName] = {
- _id: `${datasourceId}${SEPARATOR}${tableName}`,
- // TODO: this needs to accommodate composite keys
- primary: keys,
- name: tableName,
- schema: {},
- }
- }
+// // table key doesn't exist yet
+// if (!tables[tableName]) {
+// tables[tableName] = {
+// _id: `${datasourceId}${SEPARATOR}${tableName}`,
+// // TODO: this needs to accommodate composite keys
+// primary: keys,
+// name: tableName,
+// schema: {},
+// }
+// }
- tables[tableName].schema[columnName] = {
- name: columnName,
- type: TYPE_MAP[column.data_type] || FIELD_TYPES.STRING,
- }
- }
- this.tables = tables
- }
+// tables[tableName].schema[columnName] = {
+// name: columnName,
+// type: TYPE_MAP[column.data_type] || FIELD_TYPES.STRING,
+// }
+// }
+// this.tables = tables
+// }
- async query(json) {
- const operation = this._operation(json).toLowerCase()
- const sql = this._query(json)
- const response = await this.client.query(sql.sql, sql.bindings)
- return response.rows.length ? response.rows : [{ [operation]: true }]
- }
-}
+// async query(json) {
+// const operation = this._operation(json).toLowerCase()
+// const sql = this._query(json)
+// const response = await this.client.query(sql.sql, sql.bindings)
+// return response.rows.length ? response.rows : [{ [operation]: true }]
+// }
+// }
-module.exports = {
- schema: SCHEMA,
- integration: PostgresPlus,
-}
+// module.exports = {
+// schema: SCHEMA,
+// integration: PostgresPlus,
+// }
diff --git a/packages/server/src/integrations/postgres.js b/packages/server/src/integrations/postgres.js
index 72b02431be..0cbb4ffa4d 100644
--- a/packages/server/src/integrations/postgres.js
+++ b/packages/server/src/integrations/postgres.js
@@ -1,9 +1,12 @@
const { Pool } = require("pg")
const { FIELD_TYPES } = require("./Integration")
const Sql = require("./base/sql")
+const { FieldTypes } = require("../constants")
+const { SEPARATOR } = require("@budibase/auth/db")
const SCHEMA = {
docs: "https://node-postgres.com",
+ plus: true,
friendlyName: "PostgreSQL",
description:
"PostgreSQL, also known as Postgres, is a free and open-source relational database management system emphasizing extensibility and SQL compliance.",
@@ -55,6 +58,19 @@ const SCHEMA = {
},
}
+const TYPE_MAP = {
+ text: FieldTypes.LONGFORM,
+ varchar: FieldTypes.STRING,
+ integer: FieldTypes.NUMBER,
+ bigint: FieldTypes.NUMBER,
+ decimal: FieldTypes.NUMBER,
+ smallint: FieldTypes.NUMBER,
+ timestamp: FieldTypes.DATETIME,
+ time: FieldTypes.DATETIME,
+ boolean: FieldTypes.BOOLEAN,
+ json: FIELD_TYPES.JSON,
+}
+
async function internalQuery(client, sql) {
try {
return await client.query(sql)
@@ -66,6 +82,19 @@ async function internalQuery(client, sql) {
class PostgresIntegration extends Sql {
static pool
+ COLUMNS_SQL =
+ "select * from information_schema.columns where table_schema = 'public'"
+
+ PRIMARY_KEYS_SQL = `
+ select tc.table_schema, tc.table_name, kc.column_name as primary_key
+ from information_schema.table_constraints tc
+ join
+ information_schema.key_column_usage kc on kc.table_name = tc.table_name
+ and kc.table_schema = tc.table_schema
+ and kc.constraint_name = tc.constraint_name
+ where tc.constraint_type = 'PRIMARY KEY';
+ `
+
constructor(config) {
super("pg")
this.config = config
@@ -82,6 +111,48 @@ class PostgresIntegration extends Sql {
this.client = this.pool
}
+ /**
+ * Fetches the tables from the postgres table and assigns them to the datasource.
+ * @param {*} datasourceId - datasourceId to fetch
+ */
+ async buildSchema(datasourceId) {
+ let keys = []
+ try {
+ const primaryKeysResponse = await this.client.query(this.PRIMARY_KEYS_SQL)
+ for (let table of primaryKeysResponse.rows) {
+ keys.push(table.column_name || table.primary_key)
+ }
+ } catch (err) {
+ // TODO: this try catch method isn't right
+ keys = ["id"]
+ }
+
+ const columnsResponse = await this.client.query(this.COLUMNS_SQL)
+ const tables = {}
+
+ for (let column of columnsResponse.rows) {
+ const tableName = column.table_name
+ const columnName = column.column_name
+
+ // table key doesn't exist yet
+ if (!tables[tableName]) {
+ tables[tableName] = {
+ _id: `${datasourceId}${SEPARATOR}${tableName}`,
+ // TODO: this needs to accommodate composite keys
+ primary: keys,
+ name: tableName,
+ schema: {},
+ }
+ }
+
+ tables[tableName].schema[columnName] = {
+ name: columnName,
+ type: TYPE_MAP[column.data_type] || FIELD_TYPES.STRING,
+ }
+ }
+ this.tables = tables
+ }
+
async create({ sql }) {
const response = await internalQuery(this.client, sql)
return response.rows.length ? response.rows : [{ created: true }]