MirrorSave
/
budibase
mirror of https://github.com/Budibase/budibase.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Revert "Merge pull request #10038 from Budibase/fix/budi-6723" (#10143) 

This reverts commit b1b18bef18, reversing
changes made to e841ccd0ed.
This commit is contained in:
Rory Powell 2023-03-29 08:48:06 +01:00 committed by GitHub
parent 8e368525e4
commit 89d19a87a7
16 changed files with 145 additions and 296 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

9
packages/builder/src/components/backend/DataTable/modals/CreateEditColumn.svelte
View File

@ -308,7 +308,7 @@
{ name: "Auto Column", type: AUTO_TYPE }, { name: "Auto Column", type: AUTO_TYPE },
] ]
} else { } else {
let fields = [ return [
FIELDS.STRING, FIELDS.STRING,
FIELDS.BARCODEQR, FIELDS.BARCODEQR,
FIELDS.LONGFORM, FIELDS.LONGFORM,
@ -316,13 +316,10 @@
FIELDS.DATETIME, FIELDS.DATETIME,
FIELDS.NUMBER, FIELDS.NUMBER,
FIELDS.BOOLEAN, FIELDS.BOOLEAN,
FIELDS.ARRAY,
FIELDS.FORMULA, FIELDS.FORMULA,
FIELDS.LINK,
] ]
// no-sql or a spreadsheet
if (!external || table.sql) {
fields = [...fields, FIELDS.LINK, FIELDS.ARRAY]
}
return fields
} }
} }

4
packages/builder/src/components/backend/DatasourceNavigator/TableIntegrationMenu/CreateExternalTableModal.svelte
View File

@ -35,9 +35,7 @@
await datasources.fetch() await datasources.fetch()
$goto(`../../table/${table._id}`) $goto(`../../table/${table._id}`)
} catch (error) { } catch (error) {
notifications.error( notifications.error("Error saving table")
`Error saving table - ${error?.message || "unknown error"}`
)
} }
} }
</script> </script>

20
packages/builder/src/components/backend/DatasourceNavigator/modals/GoogleDatasourceConfigModal.svelte
View File

@ -1,22 +1,15 @@
<script> <script>
import { ModalContent, Body, Layout, Link } from "@budibase/bbui" import { ModalContent, Body, Layout } from "@budibase/bbui"
import { IntegrationNames } from "constants/backend" import { IntegrationNames } from "constants/backend"
import cloneDeep from "lodash/cloneDeepWith" import cloneDeep from "lodash/cloneDeepWith"
import GoogleButton from "../_components/GoogleButton.svelte" import GoogleButton from "../_components/GoogleButton.svelte"
import { saveDatasource as save } from "builderStore/datasource" import { saveDatasource as save } from "builderStore/datasource"
import { organisation } from "stores/portal"
import { onMount } from "svelte"
export let integration export let integration
export let modal export let modal
// kill the reference so the input isn't saved // kill the reference so the input isn't saved
let datasource = cloneDeep(integration) let datasource = cloneDeep(integration)
$: isGoogleConfigured = !!$organisation.google
onMount(async () => {
await organisation.init()
})
</script> </script>
<ModalContent <ModalContent
@ -25,21 +18,12 @@
cancelText="Back" cancelText="Back"
size="L" size="L"
> >
<!-- check true and false directly, don't render until flag is set -->
{#if isGoogleConfigured === true}
<Layout noPadding> <Layout noPadding>
<Body size="S" <Body size="XS"
>Authenticate with your google account to use the {IntegrationNames[ >Authenticate with your google account to use the {IntegrationNames[
datasource.type datasource.type
]} integration.</Body ]} integration.</Body
> >
</Layout> </Layout>
<GoogleButton preAuthStep={() => save(datasource, true)} /> <GoogleButton preAuthStep={() => save(datasource, true)} />
{:else if isGoogleConfigured === false}
<Body size="S"
>Google authentication is not enabled, please complete Google SSO
configuration.</Body
>
<Link href="/builder/portal/settings/auth">Configure Google SSO</Link>
{/if}
</ModalContent> </ModalContent>

17
packages/builder/src/pages/builder/portal/settings/auth/index.svelte
View File

@ -47,9 +47,8 @@
$: googleCallbackTooltip = $admin.cloud $: googleCallbackTooltip = $admin.cloud
? null ? null
: googleCallbackReadonly : googleCallbackReadonly
? "Visit the organisation page to update the platform URL" ? "Vist the organisation page to update the platform URL"
: "Leave blank to use the default callback URL" : "Leave blank to use the default callback URL"
$: googleSheetsCallbackUrl = `${$organisation.platformUrl}/api/global/auth/datasource/google/callback`
$: GoogleConfigFields = { $: GoogleConfigFields = {
Google: [ Google: [
@ -63,14 +62,6 @@
placeholder: $organisation.googleCallbackUrl, placeholder: $organisation.googleCallbackUrl,
copyButton: true, copyButton: true,
}, },
{
name: "sheetsURL",
label: "Sheets URL",
readonly: googleCallbackReadonly,
tooltip: googleCallbackTooltip,
placeholder: googleSheetsCallbackUrl,
copyButton: true,
},
], ],
} }
@ -405,11 +396,7 @@
</Heading> </Heading>
<Body size="S"> <Body size="S">
To allow users to authenticate using their Google accounts, fill out the To allow users to authenticate using their Google accounts, fill out the
fields below. Read the <Link fields below.
size="M"
href={"https://docs.budibase.com/docs/sso-with-google"}
>documentation</Link
> for more information.
</Body> </Body>
</Layout> </Layout>
<Layout gap="XS" noPadding> <Layout gap="XS" noPadding>

3
packages/server/src/api/controllers/datasource.ts
View File

@ -84,9 +84,8 @@ export async function buildSchemaFromDb(ctx: UserCtx) {
setDefaultDisplayColumns(datasource) setDefaultDisplayColumns(datasource)
const dbResp = await db.put(datasource) const dbResp = await db.put(datasource)
datasource._rev = dbResp.rev datasource._rev = dbResp.rev
const cleanedDatasource = await sdk.datasources.removeSecretSingle(datasource)
const response: any = { datasource: cleanedDatasource } const response: any = { datasource }
if (error) { if (error) {
response.error = error response.error = error
} }

42
packages/server/src/api/controllers/row/external.ts
View File

@ -12,7 +12,7 @@ import * as exporters from "../view/exporters"
import { apiFileReturn } from "../../../utilities/fileSystem" import { apiFileReturn } from "../../../utilities/fileSystem"
import { import {
Operation, Operation,
UserCtx, BBContext,
Row, Row,
PaginationJson, PaginationJson,
Table, Table,
@ -21,7 +21,6 @@ import {
SortJson, SortJson,
} from "@budibase/types" } from "@budibase/types"
import sdk from "../../../sdk" import sdk from "../../../sdk"
import * as utils from "./utils"
const { cleanExportRows } = require("./utils") const { cleanExportRows } = require("./utils")
@ -50,19 +49,12 @@ export async function handleRequest(
) )
} }
export async function patch(ctx: UserCtx) { export async function patch(ctx: BBContext) {
const inputs = ctx.request.body const inputs = ctx.request.body
const tableId = ctx.params.tableId const tableId = ctx.params.tableId
const id = inputs._id const id = inputs._id
// don't save the ID to db // don't save the ID to db
delete inputs._id delete inputs._id
const validateResult = await utils.validate({
row: inputs,
tableId,
})
if (!validateResult.valid) {
throw { validation: validateResult.errors }
}
return handleRequest(Operation.UPDATE, tableId, { return handleRequest(Operation.UPDATE, tableId, {
id: breakRowIdField(id), id: breakRowIdField(id),
row: inputs, row: inputs,
@ -70,23 +62,16 @@ export async function patch(ctx: UserCtx) {
}) })
} }
export async function save(ctx: UserCtx) { export async function save(ctx: BBContext) {
const inputs = ctx.request.body const inputs = ctx.request.body
const tableId = ctx.params.tableId const tableId = ctx.params.tableId
const validateResult = await utils.validate({
row: inputs,
tableId,
})
if (!validateResult.valid) {
throw { validation: validateResult.errors }
}
return handleRequest(Operation.CREATE, tableId, { return handleRequest(Operation.CREATE, tableId, {
row: inputs, row: inputs,
includeSqlRelationships: IncludeRelationship.EXCLUDE, includeSqlRelationships: IncludeRelationship.EXCLUDE,
}) })
} }
export async function fetchView(ctx: UserCtx) { export async function fetchView(ctx: BBContext) {
// there are no views in external datasources, shouldn't ever be called // there are no views in external datasources, shouldn't ever be called
// for now just fetch // for now just fetch
const split = ctx.params.viewName.split("all_") const split = ctx.params.viewName.split("all_")
@ -94,14 +79,14 @@ export async function fetchView(ctx: UserCtx) {
return fetch(ctx) return fetch(ctx)
} }
export async function fetch(ctx: UserCtx) { export async function fetch(ctx: BBContext) {
const tableId = ctx.params.tableId const tableId = ctx.params.tableId
return handleRequest(Operation.READ, tableId, { return handleRequest(Operation.READ, tableId, {
includeSqlRelationships: IncludeRelationship.INCLUDE, includeSqlRelationships: IncludeRelationship.INCLUDE,
}) })
} }
export async function find(ctx: UserCtx) { export async function find(ctx: BBContext) {
const id = ctx.params.rowId const id = ctx.params.rowId
const tableId = ctx.params.tableId const tableId = ctx.params.tableId
const response = (await handleRequest(Operation.READ, tableId, { const response = (await handleRequest(Operation.READ, tableId, {
@ -111,7 +96,7 @@ export async function find(ctx: UserCtx) {
return response ? response[0] : response return response ? response[0] : response
} }
export async function destroy(ctx: UserCtx) { export async function destroy(ctx: BBContext) {
const tableId = ctx.params.tableId const tableId = ctx.params.tableId
const id = ctx.request.body._id const id = ctx.request.body._id
const { row } = (await handleRequest(Operation.DELETE, tableId, { const { row } = (await handleRequest(Operation.DELETE, tableId, {
@ -121,7 +106,7 @@ export async function destroy(ctx: UserCtx) {
return { response: { ok: true }, row } return { response: { ok: true }, row }
} }
export async function bulkDestroy(ctx: UserCtx) { export async function bulkDestroy(ctx: BBContext) {
const { rows } = ctx.request.body const { rows } = ctx.request.body
const tableId = ctx.params.tableId const tableId = ctx.params.tableId
let promises: Promise<Row[] | { row: Row; table: Table }>[] = [] let promises: Promise<Row[] | { row: Row; table: Table }>[] = []
@ -137,7 +122,7 @@ export async function bulkDestroy(ctx: UserCtx) {
return { response: { ok: true }, rows: responses.map(resp => resp.row) } return { response: { ok: true }, rows: responses.map(resp => resp.row) }
} }
export async function search(ctx: UserCtx) { export async function search(ctx: BBContext) {
const tableId = ctx.params.tableId const tableId = ctx.params.tableId
const { paginate, query, ...params } = ctx.request.body const { paginate, query, ...params } = ctx.request.body
let { bookmark, limit } = params let { bookmark, limit } = params
@ -200,7 +185,12 @@ export async function search(ctx: UserCtx) {
} }
} }
export async function exportRows(ctx: UserCtx) { export async function validate(ctx: BBContext) {
// can't validate external right now - maybe in future
return { valid: true }
}
export async function exportRows(ctx: BBContext) {
const { datasourceId, tableName } = breakExternalTableId(ctx.params.tableId) const { datasourceId, tableName } = breakExternalTableId(ctx.params.tableId)
const format = ctx.query.format const format = ctx.query.format
const { columns } = ctx.request.body const { columns } = ctx.request.body
@ -254,7 +244,7 @@ export async function exportRows(ctx: UserCtx) {
return apiFileReturn(exporter(headers, exportRows)) return apiFileReturn(exporter(headers, exportRows))
} }
export async function fetchEnrichedRow(ctx: UserCtx) { export async function fetchEnrichedRow(ctx: BBContext) {
const id = ctx.params.rowId const id = ctx.params.rowId
const tableId = ctx.params.tableId const tableId = ctx.params.tableId
const { datasourceId, tableName } = breakExternalTableId(tableId) const { datasourceId, tableName } = breakExternalTableId(tableId)

9
packages/server/src/api/controllers/row/index.ts
View File

@ -2,8 +2,6 @@ import { quotas } from "@budibase/pro"
import * as internal from "./internal" import * as internal from "./internal"
import * as external from "./external" import * as external from "./external"
import { isExternalTable } from "../../../integrations/utils" import { isExternalTable } from "../../../integrations/utils"
import { Ctx } from "@budibase/types"
import * as utils from "./utils"
function pickApi(tableId: any) { function pickApi(tableId: any) {
if (isExternalTable(tableId)) { if (isExternalTable(tableId)) {
@ -131,12 +129,9 @@ export async function search(ctx: any) {
}) })
} }
export async function validate(ctx: Ctx) { export async function validate(ctx: any) {
const tableId = getTableId(ctx) const tableId = getTableId(ctx)
ctx.body = await utils.validate({ ctx.body = await pickApi(tableId).validate(ctx)
row: ctx.request.body,
tableId,
})
} }
export async function fetchEnrichedRow(ctx: any) { export async function fetchEnrichedRow(ctx: any) {

7
packages/server/src/api/controllers/row/internal.ts
View File

@ -387,6 +387,13 @@ export async function search(ctx: Ctx) {
return response return response
} }
export async function validate(ctx: Ctx) {
return utils.validate({
tableId: ctx.params.tableId,
row: ctx.request.body,
})
}
export async function exportRows(ctx: Ctx) { export async function exportRows(ctx: Ctx) {
const db = context.getAppDB() const db = context.getAppDB()
const table = await db.get(ctx.params.tableId) const table = await db.get(ctx.params.tableId)

7
packages/server/src/api/controllers/row/utils.ts
View File

@ -4,11 +4,11 @@ import { FieldTypes } from "../../../constants"
import { context } from "@budibase/backend-core" import { context } from "@budibase/backend-core"
import { makeExternalQuery } from "../../../integrations/base/query" import { makeExternalQuery } from "../../../integrations/base/query"
import { Row, Table } from "@budibase/types" import { Row, Table } from "@budibase/types"
const validateJs = require("validate.js")
const { cloneDeep } = require("lodash/fp")
import { Format } from "../view/exporters" import { Format } from "../view/exporters"
import { Ctx } from "@budibase/types" import { Ctx } from "@budibase/types"
import sdk from "../../../sdk" import sdk from "../../../sdk"
const validateJs = require("validate.js")
const { cloneDeep } = require("lodash/fp")
validateJs.extend(validateJs.validators.datetime, { validateJs.extend(validateJs.validators.datetime, {
parse: function (value: string) { parse: function (value: string) {
@ -56,7 +56,8 @@ export async function validate({
}) { }) {
let fetchedTable: Table let fetchedTable: Table
if (!table) { if (!table) {
fetchedTable = await sdk.tables.getTable(tableId) const db = context.getAppDB()
fetchedTable = await db.get(tableId)
} else { } else {
fetchedTable = table fetchedTable = table
} }

41
packages/server/src/api/controllers/table/external.ts
View File

@ -7,7 +7,6 @@ import {
generateJunctionTableName, generateJunctionTableName,
foreignKeyStructure, foreignKeyStructure,
hasTypeChanged, hasTypeChanged,
setStaticSchemas,
} from "./utils" } from "./utils"
import { FieldTypes } from "../../../constants" import { FieldTypes } from "../../../constants"
import { makeExternalQuery } from "../../../integrations/base/query" import { makeExternalQuery } from "../../../integrations/base/query"
@ -21,7 +20,7 @@ import {
Operation, Operation,
RenameColumn, RenameColumn,
FieldSchema, FieldSchema,
UserCtx, BBContext,
TableRequest, TableRequest,
RelationshipTypes, RelationshipTypes,
} from "@budibase/types" } from "@budibase/types"
@ -195,20 +194,20 @@ function isRelationshipSetup(column: FieldSchema) {
return column.foreignKey || column.through return column.foreignKey || column.through
} }
export async function save(ctx: UserCtx) { export async function save(ctx: BBContext) {
const inputs: TableRequest = ctx.request.body const table: TableRequest = ctx.request.body
const renamed = inputs?._rename const renamed = table?._rename
// can't do this right now // can't do this right now
delete inputs.rows delete table.rows
const datasourceId = getDatasourceId(ctx.request.body)! const datasourceId = getDatasourceId(ctx.request.body)!
// table doesn't exist already, note that it is created // table doesn't exist already, note that it is created
if (!inputs._id) { if (!table._id) {
inputs.created = true table.created = true
} }
let tableToSave: TableRequest = { let tableToSave: TableRequest = {
type: "table", type: "table",
_id: buildExternalTableId(datasourceId, inputs.name), _id: buildExternalTableId(datasourceId, table.name),
...inputs, ...table,
} }
let oldTable let oldTable
@ -225,10 +224,6 @@ export async function save(ctx: UserCtx) {
if (!datasource.entities) { if (!datasource.entities) {
datasource.entities = {} datasource.entities = {}
} }
// GSheets is a specific case - only ever has a static primary key
tableToSave = setStaticSchemas(datasource, tableToSave)
const oldTables = cloneDeep(datasource.entities) const oldTables = cloneDeep(datasource.entities)
const tables: Record<string, Table> = datasource.entities const tables: Record<string, Table> = datasource.entities
@ -251,7 +246,7 @@ export async function save(ctx: UserCtx) {
const junctionTable = generateManyLinkSchema( const junctionTable = generateManyLinkSchema(
datasource, datasource,
schema, schema,
tableToSave, table,
relatedTable relatedTable
) )
if (tables[junctionTable.name]) { if (tables[junctionTable.name]) {
@ -261,12 +256,10 @@ export async function save(ctx: UserCtx) {
extraTablesToUpdate.push(junctionTable) extraTablesToUpdate.push(junctionTable)
} else { } else {
const fkTable = const fkTable =
relationType === RelationshipTypes.ONE_TO_MANY relationType === RelationshipTypes.ONE_TO_MANY ? table : relatedTable
? tableToSave
: relatedTable
const foreignKey = generateLinkSchema( const foreignKey = generateLinkSchema(
schema, schema,
tableToSave, table,
relatedTable, relatedTable,
relationType relationType
) )
@ -278,11 +271,11 @@ export async function save(ctx: UserCtx) {
fkTable.constrained.push(foreignKey) fkTable.constrained.push(foreignKey)
} }
// foreign key is in other table, need to save it to external // foreign key is in other table, need to save it to external
if (fkTable._id !== tableToSave._id) { if (fkTable._id !== table._id) {
extraTablesToUpdate.push(fkTable) extraTablesToUpdate.push(fkTable)
} }
} }
generateRelatedSchema(schema, relatedTable, tableToSave, relatedColumnName) generateRelatedSchema(schema, relatedTable, table, relatedColumnName)
schema.main = true schema.main = true
} }
@ -320,7 +313,7 @@ export async function save(ctx: UserCtx) {
return tableToSave return tableToSave
} }
export async function destroy(ctx: UserCtx) { export async function destroy(ctx: BBContext) {
const tableToDelete: TableRequest = await sdk.tables.getTable( const tableToDelete: TableRequest = await sdk.tables.getTable(
ctx.params.tableId ctx.params.tableId
) )
@ -346,7 +339,7 @@ export async function destroy(ctx: UserCtx) {
return tableToDelete return tableToDelete
} }
export async function bulkImport(ctx: UserCtx) { export async function bulkImport(ctx: BBContext) {
const table = await sdk.tables.getTable(ctx.params.tableId) const table = await sdk.tables.getTable(ctx.params.tableId)
const { rows }: { rows: unknown } = ctx.request.body const { rows }: { rows: unknown } = ctx.request.body
const schema: unknown = table.schema const schema: unknown = table.schema
@ -355,7 +348,7 @@ export async function bulkImport(ctx: UserCtx) {
ctx.throw(400, "Provided data import information is invalid.") ctx.throw(400, "Provided data import information is invalid.")
} }
const parsedRows = parse(rows, schema) const parsedRows = await parse(rows, schema)
await handleRequest(Operation.BULK_CREATE, table._id!, { await handleRequest(Operation.BULK_CREATE, table._id!, {
rows: parsedRows, rows: parsedRows,
}) })

18
packages/server/src/api/controllers/table/index.ts
View File

@ -8,7 +8,7 @@ import {
import { isExternalTable, isSQL } from "../../../integrations/utils" import { isExternalTable, isSQL } from "../../../integrations/utils"
import { getDatasourceParams } from "../../../db/utils" import { getDatasourceParams } from "../../../db/utils"
import { context, events } from "@budibase/backend-core" import { context, events } from "@budibase/backend-core"
import { Table, UserCtx } from "@budibase/types" import { Table, BBContext } from "@budibase/types"
import sdk from "../../../sdk" import sdk from "../../../sdk"
import csv from "csvtojson" import csv from "csvtojson"
@ -25,7 +25,7 @@ function pickApi({ tableId, table }: { tableId?: string; table?: Table }) {
} }
// covers both internal and external // covers both internal and external
export async function fetch(ctx: UserCtx) { export async function fetch(ctx: BBContext) {
const db = context.getAppDB() const db = context.getAppDB()
const internal = await sdk.tables.getAllInternalTables() const internal = await sdk.tables.getAllInternalTables()
@ -53,12 +53,12 @@ export async function fetch(ctx: UserCtx) {
ctx.body = [...internal, ...external] ctx.body = [...internal, ...external]
} }
export async function find(ctx: UserCtx) { export async function find(ctx: BBContext) {
const tableId = ctx.params.tableId const tableId = ctx.params.tableId
ctx.body = await sdk.tables.getTable(tableId) ctx.body = await sdk.tables.getTable(tableId)
} }
export async function save(ctx: UserCtx) { export async function save(ctx: BBContext) {
const appId = ctx.appId const appId = ctx.appId
const table = ctx.request.body const table = ctx.request.body
const isImport = table.rows const isImport = table.rows
@ -79,7 +79,7 @@ export async function save(ctx: UserCtx) {
ctx.body = savedTable ctx.body = savedTable
} }
export async function destroy(ctx: UserCtx) { export async function destroy(ctx: BBContext) {
const appId = ctx.appId const appId = ctx.appId
const tableId = ctx.params.tableId const tableId = ctx.params.tableId
const deletedTable = await pickApi({ tableId }).destroy(ctx) const deletedTable = await pickApi({ tableId }).destroy(ctx)
@ -91,7 +91,7 @@ export async function destroy(ctx: UserCtx) {
ctx.body = { message: `Table ${tableId} deleted.` } ctx.body = { message: `Table ${tableId} deleted.` }
} }
export async function bulkImport(ctx: UserCtx) { export async function bulkImport(ctx: BBContext) {
const tableId = ctx.params.tableId const tableId = ctx.params.tableId
await pickApi({ tableId }).bulkImport(ctx) await pickApi({ tableId }).bulkImport(ctx)
// right now we don't trigger anything for bulk import because it // right now we don't trigger anything for bulk import because it
@ -101,7 +101,7 @@ export async function bulkImport(ctx: UserCtx) {
ctx.body = { message: `Bulk rows created.` } ctx.body = { message: `Bulk rows created.` }
} }
export async function csvToJson(ctx: UserCtx) { export async function csvToJson(ctx: BBContext) {
const { csvString } = ctx.request.body const { csvString } = ctx.request.body
const result = await csv().fromString(csvString) const result = await csv().fromString(csvString)
@ -110,7 +110,7 @@ export async function csvToJson(ctx: UserCtx) {
ctx.body = result ctx.body = result
} }
export async function validateNewTableImport(ctx: UserCtx) { export async function validateNewTableImport(ctx: BBContext) {
const { rows, schema }: { rows: unknown; schema: unknown } = ctx.request.body const { rows, schema }: { rows: unknown; schema: unknown } = ctx.request.body
if (isRows(rows) && isSchema(schema)) { if (isRows(rows) && isSchema(schema)) {
@ -121,7 +121,7 @@ export async function validateNewTableImport(ctx: UserCtx) {
} }
} }
export async function validateExistingTableImport(ctx: UserCtx) { export async function validateExistingTableImport(ctx: BBContext) {
const { rows, tableId }: { rows: unknown; tableId: unknown } = const { rows, tableId }: { rows: unknown; tableId: unknown } =
ctx.request.body ctx.request.body

20
packages/server/src/api/controllers/table/utils.ts
View File

@ -1,11 +1,7 @@
import { parse, isSchema, isRows } from "../../../utilities/schema" import { parse, isSchema, isRows } from "../../../utilities/schema"
import { getRowParams, generateRowID, InternalTables } from "../../../db/utils" import { getRowParams, generateRowID, InternalTables } from "../../../db/utils"
import { isEqual } from "lodash" import { isEqual } from "lodash"
import { import { AutoFieldSubTypes, FieldTypes } from "../../../constants"
AutoFieldSubTypes,
FieldTypes,
GOOGLE_SHEETS_PRIMARY_KEY,
} from "../../../constants"
import { import {
inputProcessing, inputProcessing,
cleanupAttachments, cleanupAttachments,
@ -20,7 +16,7 @@ import viewTemplate from "../view/viewBuilder"
import { cloneDeep } from "lodash/fp" import { cloneDeep } from "lodash/fp"
import { quotas } from "@budibase/pro" import { quotas } from "@budibase/pro"
import { events, context } from "@budibase/backend-core" import { events, context } from "@budibase/backend-core"
import { Database, Datasource, SourceName, Table } from "@budibase/types" import { Database } from "@budibase/types"
export async function clearColumns(table: any, columnNames: any) { export async function clearColumns(table: any, columnNames: any) {
const db: Database = context.getAppDB() const db: Database = context.getAppDB()
@ -396,17 +392,5 @@ export function hasTypeChanged(table: any, oldTable: any) {
return false return false
} }
// used for external tables, some of them will have static schemas that need
// to be hard set
export function setStaticSchemas(datasource: Datasource, table: Table) {
// GSheets is a specific case - only ever has a static primary key
if (table && datasource.source === SourceName.GOOGLE_SHEETS) {
table.primary = [GOOGLE_SHEETS_PRIMARY_KEY]
// if there is an id column, remove it, should never exist in GSheets
delete table.schema?.id
}
return table
}
const _TableSaveFunctions = TableSaveFunctions const _TableSaveFunctions = TableSaveFunctions
export { _TableSaveFunctions as TableSaveFunctions } export { _TableSaveFunctions as TableSaveFunctions }

1
packages/server/src/constants/index.ts
View File

@ -180,4 +180,3 @@ export enum AutomationErrors {
// pass through the list from the auth/core lib // pass through the list from the auth/core lib
export const ObjectStoreBuckets = objectStore.ObjectStoreBuckets export const ObjectStoreBuckets = objectStore.ObjectStoreBuckets
export const MAX_AUTOMATION_RECURRING_ERRORS = 5 export const MAX_AUTOMATION_RECURRING_ERRORS = 5
export const GOOGLE_SHEETS_PRIMARY_KEY = "rowNumber"

180
packages/server/src/integrations/googlesheets.ts
View File

@ -1,25 +1,22 @@
import { import {
DatasourceFieldType, DatasourceFieldType,
DatasourcePlus, DatasourcePlus,
FieldType,
Integration, Integration,
Operation,
PaginationJson, PaginationJson,
QueryJson, QueryJson,
QueryType, QueryType,
Row,
SearchFilters, SearchFilters,
SortJson, SortJson,
Table, Table,
TableRequest, TableSchema,
} from "@budibase/types" } from "@budibase/types"
import { OAuth2Client } from "google-auth-library" import { OAuth2Client } from "google-auth-library"
import { buildExternalTableId, finaliseExternalTables } from "./utils" import { buildExternalTableId } from "./utils"
import { DataSourceOperation, FieldTypes } from "../constants"
import { GoogleSpreadsheet } from "google-spreadsheet" import { GoogleSpreadsheet } from "google-spreadsheet"
import fetch from "node-fetch" import fetch from "node-fetch"
import { configs, HTTPError } from "@budibase/backend-core" import { configs, HTTPError } from "@budibase/backend-core"
import { dataFilters } from "@budibase/shared-core" import { dataFilters } from "@budibase/shared-core"
import { GOOGLE_SHEETS_PRIMARY_KEY } from "../constants"
interface GoogleSheetsConfig { interface GoogleSheetsConfig {
spreadsheetId: string spreadsheetId: string
@ -42,17 +39,6 @@ interface AuthTokenResponse {
access_token: string access_token: string
} }
const ALLOWED_TYPES = [
FieldType.STRING,
FieldType.FORMULA,
FieldType.NUMBER,
FieldType.LONGFORM,
FieldType.DATETIME,
FieldType.OPTIONS,
FieldType.BOOLEAN,
FieldType.BARCODEQR,
]
const SCHEMA: Integration = { const SCHEMA: Integration = {
plus: true, plus: true,
auth: { auth: {
@ -213,90 +199,73 @@ class GoogleSheetsIntegration implements DatasourcePlus {
this.client.useOAuth2Client(oauthClient) this.client.useOAuth2Client(oauthClient)
await this.client.loadInfo() await this.client.loadInfo()
} catch (err: any) { } catch (err) {
// this happens for xlsx imports
if (err.message?.includes("operation is not supported")) {
err.message =
"This operation is not supported - XLSX sheets must be converted."
}
console.error("Error connecting to google sheets", err) console.error("Error connecting to google sheets", err)
throw err throw err
} }
} }
getTableSchema(title: string, headerValues: string[], id?: string) { async buildSchema(datasourceId: string) {
// base table
const table: Table = {
name: title,
primary: [GOOGLE_SHEETS_PRIMARY_KEY],
schema: {},
}
if (id) {
table._id = id
}
// build schema from headers
for (let header of headerValues) {
table.schema[header] = {
name: header,
type: FieldType.STRING,
}
}
return table
}
async buildSchema(datasourceId: string, entities: Record<string, Table>) {
await this.connect() await this.connect()
const sheets = this.client.sheetsByIndex const sheets = this.client.sheetsByIndex
const tables: Record<string, Table> = {} const tables: Record<string, Table> = {}
for (let sheet of sheets) { for (let sheet of sheets) {
// must fetch rows to determine schema // must fetch rows to determine schema
await sheet.getRows() await sheet.getRows()
// build schema
const schema: TableSchema = {}
const id = buildExternalTableId(datasourceId, sheet.title) // build schema from headers
tables[sheet.title] = this.getTableSchema( for (let header of sheet.headerValues) {
sheet.title, schema[header] = {
sheet.headerValues, name: header,
id type: FieldTypes.STRING,
)
} }
const final = finaliseExternalTables(tables, entities) }
this.tables = final.tables
this.schemaErrors = final.errors // create tables
tables[sheet.title] = {
_id: buildExternalTableId(datasourceId, sheet.title),
name: sheet.title,
primary: ["rowNumber"],
schema,
}
}
this.tables = tables
} }
async query(json: QueryJson) { async query(json: QueryJson) {
const sheet = json.endpoint.entityId const sheet = json.endpoint.entityId
switch (json.endpoint.operation) {
case Operation.CREATE: const handlers = {
return this.create({ sheet, row: json.body as Row }) [DataSourceOperation.CREATE]: () =>
case Operation.BULK_CREATE: this.create({ sheet, row: json.body }),
return this.createBulk({ sheet, rows: json.body as Row[] }) [DataSourceOperation.READ]: () => this.read({ ...json, sheet }),
case Operation.READ: [DataSourceOperation.UPDATE]: () =>
return this.read({ ...json, sheet }) this.update({
case Operation.UPDATE:
return this.update({
// exclude the header row and zero index // exclude the header row and zero index
rowIndex: json.extra?.idFilter?.equal?.rowNumber - 2, rowIndex: json.extra?.idFilter?.equal?.rowNumber - 2,
sheet, sheet,
row: json.body, row: json.body,
}) }),
case Operation.DELETE: [DataSourceOperation.DELETE]: () =>
return this.delete({ this.delete({
// exclude the header row and zero index // exclude the header row and zero index
rowIndex: json.extra?.idFilter?.equal?.rowNumber - 2, rowIndex: json.extra?.idFilter?.equal?.rowNumber - 2,
sheet, sheet,
}) }),
case Operation.CREATE_TABLE: [DataSourceOperation.CREATE_TABLE]: () =>
return this.createTable(json?.table?.name) this.createTable(json?.table?.name),
case Operation.UPDATE_TABLE: [DataSourceOperation.UPDATE_TABLE]: () => this.updateTable(json.table),
return this.updateTable(json.table!) [DataSourceOperation.DELETE_TABLE]: () =>
case Operation.DELETE_TABLE: this.deleteTable(json?.table?.name),
return this.deleteTable(json?.table?.name)
default:
throw new Error(
`GSheets integration does not support "${json.endpoint.operation}".`
)
} }
// @ts-ignore
const internalQueryMethod = handlers[json.endpoint.operation]
return await internalQueryMethod()
} }
buildRowObject(headers: string[], values: string[], rowNumber: number) { buildRowObject(headers: string[], values: string[], rowNumber: number) {
@ -309,19 +278,17 @@ class GoogleSheetsIntegration implements DatasourcePlus {
} }
async createTable(name?: string) { async createTable(name?: string) {
if (!name) {
throw new Error("Must provide name for new sheet.")
}
try { try {
await this.connect() await this.connect()
return await this.client.addSheet({ title: name, headerValues: [name] }) return await this.client.addSheet({ title: name, headerValues: ["test"] })
} catch (err) { } catch (err) {
console.error("Error creating new table in google sheets", err) console.error("Error creating new table in google sheets", err)
throw err throw err
} }
} }
async updateTable(table: TableRequest) { async updateTable(table?: any) {
try {
await this.connect() await this.connect()
const sheet = this.client.sheetsByTitle[table.name] const sheet = this.client.sheetsByTitle[table.name]
await sheet.loadHeaderRow() await sheet.loadHeaderRow()
@ -335,46 +302,25 @@ class GoogleSheetsIntegration implements DatasourcePlus {
headers.push(header) headers.push(header)
} }
} }
try {
await sheet.setHeaderRow(headers) await sheet.setHeaderRow(headers)
} catch (err) {
console.error("Error updating column name in google sheets", err)
throw err
}
} else { } else {
const updatedHeaderValues = [...sheet.headerValues] const updatedHeaderValues = [...sheet.headerValues]
// add new column - doesn't currently exist const newField = Object.keys(table.schema).find(
for (let [key, column] of Object.entries(table.schema)) { key => !sheet.headerValues.includes(key)
if (!ALLOWED_TYPES.includes(column.type)) {
throw new Error(
`Column type: ${column.type} not allowed for GSheets integration.`
) )
}
if ( if (newField) {
!sheet.headerValues.includes(key) && updatedHeaderValues.push(newField)
column.type !== FieldType.FORMULA
) {
updatedHeaderValues.push(key)
}
} }
// clear out deleted columns
for (let key of sheet.headerValues) {
if (!Object.keys(table.schema).includes(key)) {
const idx = updatedHeaderValues.indexOf(key)
updatedHeaderValues.splice(idx, 1)
}
}
try {
await sheet.setHeaderRow(updatedHeaderValues) await sheet.setHeaderRow(updatedHeaderValues)
}
} catch (err) { } catch (err) {
console.error("Error updating table in google sheets", err) console.error("Error updating table in google sheets", err)
throw err throw err
} }
} }
}
async deleteTable(sheet: any) { async deleteTable(sheet: any) {
try { try {
@ -403,24 +349,6 @@ class GoogleSheetsIntegration implements DatasourcePlus {
} }
} }
async createBulk(query: { sheet: string; rows: any[] }) {
try {
await this.connect()
const sheet = this.client.sheetsByTitle[query.sheet]
let rowsToInsert = []
for (let row of query.rows) {
rowsToInsert.push(typeof row === "string" ? JSON.parse(row) : row)
}
const rows = await sheet.addRows(rowsToInsert)
return rows.map(row =>
this.buildRowObject(sheet.headerValues, row._rawData, row._rowNumber)
)
} catch (err) {
console.error("Error bulk writing to google sheets", err)
throw err
}
}
async read(query: { async read(query: {
sheet: string sheet: string
filters?: SearchFilters filters?: SearchFilters

20
packages/server/src/integrations/utils.ts
View File

@ -4,7 +4,6 @@ import { FieldTypes, BuildSchemaErrors, InvalidColumns } from "../constants"
const DOUBLE_SEPARATOR = `${SEPARATOR}${SEPARATOR}` const DOUBLE_SEPARATOR = `${SEPARATOR}${SEPARATOR}`
const ROW_ID_REGEX = /^\[.*]$/g const ROW_ID_REGEX = /^\[.*]$/g
const ENCODED_SPACE = encodeURIComponent(" ")
const SQL_NUMBER_TYPE_MAP = { const SQL_NUMBER_TYPE_MAP = {
integer: FieldTypes.NUMBER, integer: FieldTypes.NUMBER,
@ -80,10 +79,6 @@ export function isExternalTable(tableId: string) {
} }
export function buildExternalTableId(datasourceId: string, tableName: string) { export function buildExternalTableId(datasourceId: string, tableName: string) {
// encode spaces
if (tableName.includes(" ")) {
tableName = encodeURIComponent(tableName)
}
return `${datasourceId}${DOUBLE_SEPARATOR}${tableName}` return `${datasourceId}${DOUBLE_SEPARATOR}${tableName}`
} }
@ -95,10 +90,6 @@ export function breakExternalTableId(tableId: string | undefined) {
let datasourceId = parts.shift() let datasourceId = parts.shift()
// if they need joined // if they need joined
let tableName = parts.join(DOUBLE_SEPARATOR) let tableName = parts.join(DOUBLE_SEPARATOR)
// if contains encoded spaces, decode it
if (tableName.includes(ENCODED_SPACE)) {
tableName = decodeURIComponent(tableName)
}
return { datasourceId, tableName } return { datasourceId, tableName }
} }
@ -209,9 +200,9 @@ export function isIsoDateString(str: string) {
* @param column The column to check, to see if it is a valid relationship. * @param column The column to check, to see if it is a valid relationship.
* @param tableIds The IDs of the tables which currently exist. * @param tableIds The IDs of the tables which currently exist.
*/ */
export function shouldCopyRelationship( function shouldCopyRelationship(
column: { type: string; tableId?: string }, column: { type: string; tableId?: string },
tableIds: string[] tableIds: [string]
) { ) {
return ( return (
column.type === FieldTypes.LINK && column.type === FieldTypes.LINK &&
@ -228,7 +219,7 @@ export function shouldCopyRelationship(
* @param column The column to check for options or boolean type. * @param column The column to check for options or boolean type.
* @param fetchedColumn The fetched column to check for the type in the external database. * @param fetchedColumn The fetched column to check for the type in the external database.
*/ */
export function shouldCopySpecialColumn( function shouldCopySpecialColumn(
column: { type: string }, column: { type: string },
fetchedColumn: { type: string } | undefined fetchedColumn: { type: string } | undefined
) { ) {
@ -266,12 +257,9 @@ function copyExistingPropsOver(
tableIds: [string] tableIds: [string]
) { ) {
if (entities && entities[tableName]) { if (entities && entities[tableName]) {
if (entities[tableName]?.primaryDisplay) { if (entities[tableName].primaryDisplay) {
table.primaryDisplay = entities[tableName].primaryDisplay table.primaryDisplay = entities[tableName].primaryDisplay
} }
if (entities[tableName]?.created) {
table.created = entities[tableName]?.created
}
const existingTableSchema = entities[tableName].schema const existingTableSchema = entities[tableName].schema
for (let key in existingTableSchema) { for (let key in existingTableSchema) {
if (!existingTableSchema.hasOwnProperty(key)) { if (!existingTableSchema.hasOwnProperty(key)) {

1
packages/types/src/documents/app/table.ts
View File

@ -76,7 +76,6 @@ export interface Table extends Document {
sql?: boolean sql?: boolean
indexes?: { [key: string]: any } indexes?: { [key: string]: any }
rows?: { [key: string]: any } rows?: { [key: string]: any }
created?: boolean
} }
export interface TableRequest extends Table { export interface TableRequest extends Table {