MirrorSave
/
budibase
mirror of https://github.com/Budibase/budibase.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Merge pull request #10024 from Budibase/fix/budi-6723 

Various Googlesheet issues
This commit is contained in:
Michael Drury 2023-03-15 10:18:04 +00:00 committed by GitHub
parent df4f6c3b09 79f2e9d970
commit 6507f7a516
16 changed files with 302 additions and 151 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

9
packages/builder/src/components/backend/DataTable/modals/CreateEditColumn.svelte
View File

@ -308,7 +308,7 @@
{ name: "Auto Column", type: AUTO_TYPE },
]
} else {
return [
let fields = [
FIELDS.STRING,
FIELDS.BARCODEQR,
FIELDS.LONGFORM,
@ -316,10 +316,13 @@
FIELDS.DATETIME,
FIELDS.NUMBER,
FIELDS.BOOLEAN,
FIELDS.ARRAY,
FIELDS.FORMULA,
FIELDS.LINK,
]
// no-sql or a spreadsheet
if (!external || table.sql) {
fields = [...fields, FIELDS.LINK, FIELDS.ARRAY]
}
return fields
}
}

4
packages/builder/src/components/backend/DatasourceNavigator/TableIntegrationMenu/CreateExternalTableModal.svelte
View File

@ -35,7 +35,9 @@
await datasources.fetch()
$goto(`../../table/${table._id}`)
} catch (error) {
notifications.error("Error saving table")
notifications.error(
`Error saving table - ${error?.message || "unknown error"}`
)
}
}
</script>

32
packages/builder/src/components/backend/DatasourceNavigator/modals/GoogleDatasourceConfigModal.svelte
View File

@ -1,15 +1,22 @@
<script>
import { ModalContent, Body, Layout } from "@budibase/bbui"
import { ModalContent, Body, Layout, Link } from "@budibase/bbui"
import { IntegrationNames } from "constants/backend"
import cloneDeep from "lodash/cloneDeepWith"
import GoogleButton from "../_components/GoogleButton.svelte"
import { saveDatasource as save } from "builderStore/datasource"
import { organisation } from "stores/portal"
import { onMount } from "svelte"
export let integration
export let modal
// kill the reference so the input isn't saved
let datasource = cloneDeep(integration)
$: isGoogleConfigured = !!$organisation.google
onMount(async () => {
await organisation.init()
})
</script>
<ModalContent
@ -18,12 +25,21 @@
cancelText="Back"
size="L"
>
<Layout noPadding>
<Body size="XS"
>Authenticate with your google account to use the {IntegrationNames[
datasource.type
]} integration.</Body
<!-- check true and false directly, don't render until flag is set -->
{#if isGoogleConfigured === true}
<Layout noPadding>
<Body size="S"
>Authenticate with your google account to use the {IntegrationNames[
datasource.type
]} integration.</Body
>
</Layout>
<GoogleButton preAuthStep={() => save(datasource, true)} />
{:else if isGoogleConfigured === false}
<Body size="S"
>Google authentication is not enabled, please complete Google SSO
configuration.</Body
>
</Layout>
<GoogleButton preAuthStep={() => save(datasource, true)} />
<Link href="/builder/portal/settings/auth">Configure Google SSO</Link>
{/if}
</ModalContent>

17
packages/builder/src/pages/builder/portal/settings/auth/index.svelte
View File

@ -47,8 +47,9 @@
$: googleCallbackTooltip = $admin.cloud
? null
: googleCallbackReadonly
? "Vist the organisation page to update the platform URL"
? "Visit the organisation page to update the platform URL"
: "Leave blank to use the default callback URL"
$: googleSheetsCallbackUrl = `${$organisation.platformUrl}/api/global/auth/datasource/google/callback`
$: GoogleConfigFields = {
Google: [
@ -62,6 +63,14 @@
placeholder: $organisation.googleCallbackUrl,
copyButton: true,
},
{
name: "sheetsURL",
label: "Sheets URL",
readonly: googleCallbackReadonly,
tooltip: googleCallbackTooltip,
placeholder: googleSheetsCallbackUrl,
copyButton: true,
},
],
}
@ -396,7 +405,11 @@
</Heading>
<Body size="S">
To allow users to authenticate using their Google accounts, fill out the
fields below.
fields below. Read the <Link
size="M"
href={"https://docs.budibase.com/docs/sso-with-google"}
>documentation</Link
> for more information.
</Body>
</Layout>
<Layout gap="XS" noPadding>

3
packages/server/src/api/controllers/datasource.ts
View File

@ -84,8 +84,9 @@ export async function buildSchemaFromDb(ctx: UserCtx) {
setDefaultDisplayColumns(datasource)
const dbResp = await db.put(datasource)
datasource._rev = dbResp.rev
const cleanedDatasource = await sdk.datasources.removeSecretSingle(datasource)
const response: any = { datasource }
const response: any = { datasource: cleanedDatasource }
if (error) {
response.error = error
}

42
packages/server/src/api/controllers/row/external.ts
View File

@ -12,7 +12,7 @@ import * as exporters from "../view/exporters"
import { apiFileReturn } from "../../../utilities/fileSystem"
import {
Operation,
BBContext,
UserCtx,
Row,
PaginationJson,
Table,
@ -21,6 +21,7 @@ import {
SortJson,
} from "@budibase/types"
import sdk from "../../../sdk"
import * as utils from "./utils"
const { cleanExportRows } = require("./utils")
@ -49,12 +50,19 @@ export async function handleRequest(
)
}
export async function patch(ctx: BBContext) {
export async function patch(ctx: UserCtx) {
const inputs = ctx.request.body
const tableId = ctx.params.tableId
const id = inputs._id
// don't save the ID to db
delete inputs._id
const validateResult = await utils.validate({
row: inputs,
tableId,
})
if (!validateResult.valid) {
throw { validation: validateResult.errors }
}
return handleRequest(Operation.UPDATE, tableId, {
id: breakRowIdField(id),
row: inputs,
@ -62,16 +70,23 @@ export async function patch(ctx: BBContext) {
})
}
export async function save(ctx: BBContext) {
export async function save(ctx: UserCtx) {
const inputs = ctx.request.body
const tableId = ctx.params.tableId
const validateResult = await utils.validate({
row: inputs,
tableId,
})
if (!validateResult.valid) {
throw { validation: validateResult.errors }
}
return handleRequest(Operation.CREATE, tableId, {
row: inputs,
includeSqlRelationships: IncludeRelationship.EXCLUDE,
})
}
export async function fetchView(ctx: BBContext) {
export async function fetchView(ctx: UserCtx) {
// there are no views in external datasources, shouldn't ever be called
// for now just fetch
const split = ctx.params.viewName.split("all_")
@ -79,14 +94,14 @@ export async function fetchView(ctx: BBContext) {
return fetch(ctx)
}
export async function fetch(ctx: BBContext) {
export async function fetch(ctx: UserCtx) {
const tableId = ctx.params.tableId
return handleRequest(Operation.READ, tableId, {
includeSqlRelationships: IncludeRelationship.INCLUDE,
})
}
export async function find(ctx: BBContext) {
export async function find(ctx: UserCtx) {
const id = ctx.params.rowId
const tableId = ctx.params.tableId
const response = (await handleRequest(Operation.READ, tableId, {
@ -96,7 +111,7 @@ export async function find(ctx: BBContext) {
return response ? response[0] : response
}
export async function destroy(ctx: BBContext) {
export async function destroy(ctx: UserCtx) {
const tableId = ctx.params.tableId
const id = ctx.request.body._id
const { row } = (await handleRequest(Operation.DELETE, tableId, {
@ -106,7 +121,7 @@ export async function destroy(ctx: BBContext) {
return { response: { ok: true }, row }
}
export async function bulkDestroy(ctx: BBContext) {
export async function bulkDestroy(ctx: UserCtx) {
const { rows } = ctx.request.body
const tableId = ctx.params.tableId
let promises: Promise<Row[] | { row: Row; table: Table }>[] = []
@ -122,7 +137,7 @@ export async function bulkDestroy(ctx: BBContext) {
return { response: { ok: true }, rows: responses.map(resp => resp.row) }
}
export async function search(ctx: BBContext) {
export async function search(ctx: UserCtx) {
const tableId = ctx.params.tableId
const { paginate, query, ...params } = ctx.request.body
let { bookmark, limit } = params
@ -185,12 +200,7 @@ export async function search(ctx: BBContext) {
}
}
export async function validate(ctx: BBContext) {
// can't validate external right now - maybe in future
return { valid: true }
}
export async function exportRows(ctx: BBContext) {
export async function exportRows(ctx: UserCtx) {
const { datasourceId, tableName } = breakExternalTableId(ctx.params.tableId)
const format = ctx.query.format
const { columns } = ctx.request.body
@ -244,7 +254,7 @@ export async function exportRows(ctx: BBContext) {
return apiFileReturn(exporter(headers, exportRows))
}
export async function fetchEnrichedRow(ctx: BBContext) {
export async function fetchEnrichedRow(ctx: UserCtx) {
const id = ctx.params.rowId
const tableId = ctx.params.tableId
const { datasourceId, tableName } = breakExternalTableId(tableId)

9
packages/server/src/api/controllers/row/index.ts
View File

@ -2,6 +2,8 @@ import { quotas } from "@budibase/pro"
import * as internal from "./internal"
import * as external from "./external"
import { isExternalTable } from "../../../integrations/utils"
import { Ctx } from "@budibase/types"
import * as utils from "./utils"
function pickApi(tableId: any) {
if (isExternalTable(tableId)) {
@ -129,9 +131,12 @@ export async function search(ctx: any) {
})
}
export async function validate(ctx: any) {
export async function validate(ctx: Ctx) {
const tableId = getTableId(ctx)
ctx.body = await pickApi(tableId).validate(ctx)
ctx.body = await utils.validate({
row: ctx.request.body,
tableId,
})
}
export async function fetchEnrichedRow(ctx: any) {

7
packages/server/src/api/controllers/row/internal.ts
View File

@ -387,13 +387,6 @@ export async function search(ctx: Ctx) {
return response
}
export async function validate(ctx: Ctx) {
return utils.validate({
tableId: ctx.params.tableId,
row: ctx.request.body,
})
}
export async function exportRows(ctx: Ctx) {
const db = context.getAppDB()
const table = await db.get(ctx.params.tableId)

7
packages/server/src/api/controllers/row/utils.ts
View File

@ -4,11 +4,11 @@ import { FieldTypes } from "../../../constants"
import { context } from "@budibase/backend-core"
import { makeExternalQuery } from "../../../integrations/base/query"
import { Row, Table } from "@budibase/types"
const validateJs = require("validate.js")
const { cloneDeep } = require("lodash/fp")
import { Format } from "../view/exporters"
import { Ctx } from "@budibase/types"
import sdk from "../../../sdk"
const validateJs = require("validate.js")
const { cloneDeep } = require("lodash/fp")
validateJs.extend(validateJs.validators.datetime, {
parse: function (value: string) {
@ -56,8 +56,7 @@ export async function validate({
}) {
let fetchedTable: Table
if (!table) {
const db = context.getAppDB()
fetchedTable = await db.get(tableId)
fetchedTable = await sdk.tables.getTable(tableId)
} else {
fetchedTable = table
}

41
packages/server/src/api/controllers/table/external.ts
View File

@ -7,6 +7,7 @@ import {
generateJunctionTableName,
foreignKeyStructure,
hasTypeChanged,
setStaticSchemas,
} from "./utils"
import { FieldTypes } from "../../../constants"
import { makeExternalQuery } from "../../../integrations/base/query"
@ -20,7 +21,7 @@ import {
Operation,
RenameColumn,
FieldSchema,
BBContext,
UserCtx,
TableRequest,
RelationshipTypes,
} from "@budibase/types"
@ -194,20 +195,20 @@ function isRelationshipSetup(column: FieldSchema) {
return column.foreignKey || column.through
}
export async function save(ctx: BBContext) {
const table: TableRequest = ctx.request.body
const renamed = table?._rename
export async function save(ctx: UserCtx) {
const inputs: TableRequest = ctx.request.body
const renamed = inputs?._rename
// can't do this right now
delete table.rows
delete inputs.rows
const datasourceId = getDatasourceId(ctx.request.body)!
// table doesn't exist already, note that it is created
if (!table._id) {
table.created = true
if (!inputs._id) {
inputs.created = true
}
let tableToSave: TableRequest = {
type: "table",
_id: buildExternalTableId(datasourceId, table.name),
...table,
_id: buildExternalTableId(datasourceId, inputs.name),
...inputs,
}
let oldTable
@ -224,6 +225,10 @@ export async function save(ctx: BBContext) {
if (!datasource.entities) {
datasource.entities = {}
}
// GSheets is a specific case - only ever has a static primary key
tableToSave = setStaticSchemas(datasource, tableToSave)
const oldTables = cloneDeep(datasource.entities)
const tables: Record<string, Table> = datasource.entities
@ -246,7 +251,7 @@ export async function save(ctx: BBContext) {
const junctionTable = generateManyLinkSchema(
datasource,
schema,
table,
tableToSave,
relatedTable
)
if (tables[junctionTable.name]) {
@ -256,10 +261,12 @@ export async function save(ctx: BBContext) {
extraTablesToUpdate.push(junctionTable)
} else {
const fkTable =
relationType === RelationshipTypes.ONE_TO_MANY ? table : relatedTable
relationType === RelationshipTypes.ONE_TO_MANY
? tableToSave
: relatedTable
const foreignKey = generateLinkSchema(
schema,
table,
tableToSave,
relatedTable,
relationType
)
@ -271,11 +278,11 @@ export async function save(ctx: BBContext) {
fkTable.constrained.push(foreignKey)
}
// foreign key is in other table, need to save it to external
if (fkTable._id !== table._id) {
if (fkTable._id !== tableToSave._id) {
extraTablesToUpdate.push(fkTable)
}
}
generateRelatedSchema(schema, relatedTable, table, relatedColumnName)
generateRelatedSchema(schema, relatedTable, tableToSave, relatedColumnName)
schema.main = true
}
@ -313,7 +320,7 @@ export async function save(ctx: BBContext) {
return tableToSave
}
export async function destroy(ctx: BBContext) {
export async function destroy(ctx: UserCtx) {
const tableToDelete: TableRequest = await sdk.tables.getTable(
ctx.params.tableId
)
@ -339,7 +346,7 @@ export async function destroy(ctx: BBContext) {
return tableToDelete
}
export async function bulkImport(ctx: BBContext) {
export async function bulkImport(ctx: UserCtx) {
const table = await sdk.tables.getTable(ctx.params.tableId)
const { rows }: { rows: unknown } = ctx.request.body
const schema: unknown = table.schema
@ -348,7 +355,7 @@ export async function bulkImport(ctx: BBContext) {
ctx.throw(400, "Provided data import information is invalid.")
}
const parsedRows = await parse(rows, schema)
const parsedRows = parse(rows, schema)
await handleRequest(Operation.BULK_CREATE, table._id!, {
rows: parsedRows,
})

18
packages/server/src/api/controllers/table/index.ts
View File

@ -8,7 +8,7 @@ import {
import { isExternalTable, isSQL } from "../../../integrations/utils"
import { getDatasourceParams } from "../../../db/utils"
import { context, events } from "@budibase/backend-core"
import { Table, BBContext } from "@budibase/types"
import { Table, UserCtx } from "@budibase/types"
import sdk from "../../../sdk"
import csv from "csvtojson"
@ -25,7 +25,7 @@ function pickApi({ tableId, table }: { tableId?: string; table?: Table }) {
}
// covers both internal and external
export async function fetch(ctx: BBContext) {
export async function fetch(ctx: UserCtx) {
const db = context.getAppDB()
const internal = await sdk.tables.getAllInternalTables()
@ -53,12 +53,12 @@ export async function fetch(ctx: BBContext) {
ctx.body = [...internal, ...external]
}
export async function find(ctx: BBContext) {
export async function find(ctx: UserCtx) {
const tableId = ctx.params.tableId
ctx.body = await sdk.tables.getTable(tableId)
}
export async function save(ctx: BBContext) {
export async function save(ctx: UserCtx) {
const appId = ctx.appId
const table = ctx.request.body
const isImport = table.rows
@ -79,7 +79,7 @@ export async function save(ctx: BBContext) {
ctx.body = savedTable
}
export async function destroy(ctx: BBContext) {
export async function destroy(ctx: UserCtx) {
const appId = ctx.appId
const tableId = ctx.params.tableId
const deletedTable = await pickApi({ tableId }).destroy(ctx)
@ -91,7 +91,7 @@ export async function destroy(ctx: BBContext) {
ctx.body = { message: `Table ${tableId} deleted.` }
}
export async function bulkImport(ctx: BBContext) {
export async function bulkImport(ctx: UserCtx) {
const tableId = ctx.params.tableId
await pickApi({ tableId }).bulkImport(ctx)
// right now we don't trigger anything for bulk import because it
@ -101,7 +101,7 @@ export async function bulkImport(ctx: BBContext) {
ctx.body = { message: `Bulk rows created.` }
}
export async function csvToJson(ctx: BBContext) {
export async function csvToJson(ctx: UserCtx) {
const { csvString } = ctx.request.body
const result = await csv().fromString(csvString)
@ -110,7 +110,7 @@ export async function csvToJson(ctx: BBContext) {
ctx.body = result
}
export async function validateNewTableImport(ctx: BBContext) {
export async function validateNewTableImport(ctx: UserCtx) {
const { rows, schema }: { rows: unknown; schema: unknown } = ctx.request.body
if (isRows(rows) && isSchema(schema)) {
@ -121,7 +121,7 @@ export async function validateNewTableImport(ctx: BBContext) {
}
}
export async function validateExistingTableImport(ctx: BBContext) {
export async function validateExistingTableImport(ctx: UserCtx) {
const { rows, tableId }: { rows: unknown; tableId: unknown } =
ctx.request.body

20
packages/server/src/api/controllers/table/utils.ts
View File

@ -1,7 +1,11 @@
import { parse, isSchema, isRows } from "../../../utilities/schema"
import { getRowParams, generateRowID, InternalTables } from "../../../db/utils"
import { isEqual } from "lodash"
import { AutoFieldSubTypes, FieldTypes } from "../../../constants"
import {
AutoFieldSubTypes,
FieldTypes,
GOOGLE_SHEETS_PRIMARY_KEY,
} from "../../../constants"
import {
inputProcessing,
cleanupAttachments,
@ -16,7 +20,7 @@ import viewTemplate from "../view/viewBuilder"
import { cloneDeep } from "lodash/fp"
import { quotas } from "@budibase/pro"
import { events, context } from "@budibase/backend-core"
import { Database } from "@budibase/types"
import { Database, Datasource, SourceName, Table } from "@budibase/types"
export async function clearColumns(table: any, columnNames: any) {
const db: Database = context.getAppDB()
@ -392,5 +396,17 @@ export function hasTypeChanged(table: any, oldTable: any) {
return false
}
// used for external tables, some of them will have static schemas that need
// to be hard set
export function setStaticSchemas(datasource: Datasource, table: Table) {
// GSheets is a specific case - only ever has a static primary key
if (table && datasource.source === SourceName.GOOGLE_SHEETS) {
table.primary = [GOOGLE_SHEETS_PRIMARY_KEY]
// if there is an id column, remove it, should never exist in GSheets
delete table.schema?.id
}
return table
}
const _TableSaveFunctions = TableSaveFunctions
export { _TableSaveFunctions as TableSaveFunctions }

1
packages/server/src/constants/index.ts
View File

@ -180,3 +180,4 @@ export enum AutomationErrors {
// pass through the list from the auth/core lib
export const ObjectStoreBuckets = objectStore.ObjectStoreBuckets
export const MAX_AUTOMATION_RECURRING_ERRORS = 5
export const GOOGLE_SHEETS_PRIMARY_KEY = "rowNumber"

222
packages/server/src/integrations/googlesheets.ts
View File

@ -1,22 +1,25 @@
import {
DatasourceFieldType,
DatasourcePlus,
FieldType,
Integration,
Operation,
PaginationJson,
QueryJson,
QueryType,
Row,
SearchFilters,
SortJson,
Table,
TableSchema,
TableRequest,
} from "@budibase/types"
import { OAuth2Client } from "google-auth-library"
import { buildExternalTableId } from "./utils"
import { DataSourceOperation, FieldTypes } from "../constants"
import { buildExternalTableId, finaliseExternalTables } from "./utils"
import { GoogleSpreadsheet } from "google-spreadsheet"
import fetch from "node-fetch"
import { configs, HTTPError } from "@budibase/backend-core"
import { dataFilters } from "@budibase/shared-core"
import { GOOGLE_SHEETS_PRIMARY_KEY } from "../constants"
interface GoogleSheetsConfig {
spreadsheetId: string
@ -39,6 +42,17 @@ interface AuthTokenResponse {
access_token: string
}
const ALLOWED_TYPES = [
FieldType.STRING,
FieldType.FORMULA,
FieldType.NUMBER,
FieldType.LONGFORM,
FieldType.DATETIME,
FieldType.OPTIONS,
FieldType.BOOLEAN,
FieldType.BARCODEQR,
]
const SCHEMA: Integration = {
plus: true,
auth: {
@ -199,73 +213,90 @@ class GoogleSheetsIntegration implements DatasourcePlus {
this.client.useOAuth2Client(oauthClient)
await this.client.loadInfo()
} catch (err) {
} catch (err: any) {
// this happens for xlsx imports
if (err.message?.includes("operation is not supported")) {
err.message =
"This operation is not supported - XLSX sheets must be converted."
}
console.error("Error connecting to google sheets", err)
throw err
}
}
async buildSchema(datasourceId: string) {
getTableSchema(title: string, headerValues: string[], id?: string) {
// base table
const table: Table = {
name: title,
primary: [GOOGLE_SHEETS_PRIMARY_KEY],
schema: {},
}
if (id) {
table._id = id
}
// build schema from headers
for (let header of headerValues) {
table.schema[header] = {
name: header,
type: FieldType.STRING,
}
}
return table
}
async buildSchema(datasourceId: string, entities: Record<string, Table>) {
await this.connect()
const sheets = this.client.sheetsByIndex
const tables: Record<string, Table> = {}
for (let sheet of sheets) {
// must fetch rows to determine schema
await sheet.getRows()
// build schema
const schema: TableSchema = {}
// build schema from headers
for (let header of sheet.headerValues) {
schema[header] = {
name: header,
type: FieldTypes.STRING,
}
}
// create tables
tables[sheet.title] = {
_id: buildExternalTableId(datasourceId, sheet.title),
name: sheet.title,
primary: ["rowNumber"],
schema,
}
const id = buildExternalTableId(datasourceId, sheet.title)
tables[sheet.title] = this.getTableSchema(
sheet.title,
sheet.headerValues,
id
)
}
this.tables = tables
const final = finaliseExternalTables(tables, entities)
this.tables = final.tables
this.schemaErrors = final.errors
}
async query(json: QueryJson) {
const sheet = json.endpoint.entityId
const handlers = {
[DataSourceOperation.CREATE]: () =>
this.create({ sheet, row: json.body }),
[DataSourceOperation.READ]: () => this.read({ ...json, sheet }),
[DataSourceOperation.UPDATE]: () =>
this.update({
switch (json.endpoint.operation) {
case Operation.CREATE:
return this.create({ sheet, row: json.body as Row })
case Operation.BULK_CREATE:
return this.createBulk({ sheet, rows: json.body as Row[] })
case Operation.READ:
return this.read({ ...json, sheet })
case Operation.UPDATE:
return this.update({
// exclude the header row and zero index
rowIndex: json.extra?.idFilter?.equal?.rowNumber - 2,
sheet,
row: json.body,
}),
[DataSourceOperation.DELETE]: () =>
this.delete({
})
case Operation.DELETE:
return this.delete({
// exclude the header row and zero index
rowIndex: json.extra?.idFilter?.equal?.rowNumber - 2,
sheet,
}),
[DataSourceOperation.CREATE_TABLE]: () =>
this.createTable(json?.table?.name),
[DataSourceOperation.UPDATE_TABLE]: () => this.updateTable(json.table),
[DataSourceOperation.DELETE_TABLE]: () =>
this.deleteTable(json?.table?.name),
})
case Operation.CREATE_TABLE:
return this.createTable(json?.table?.name)
case Operation.UPDATE_TABLE:
return this.updateTable(json.table!)
case Operation.DELETE_TABLE:
return this.deleteTable(json?.table?.name)
default:
throw new Error(
`GSheets integration does not support "${json.endpoint.operation}".`
)
}
// @ts-ignore
const internalQueryMethod = handlers[json.endpoint.operation]
return await internalQueryMethod()
}
buildRowObject(headers: string[], values: string[], rowNumber: number) {
@ -278,47 +309,70 @@ class GoogleSheetsIntegration implements DatasourcePlus {
}
async createTable(name?: string) {
if (!name) {
throw new Error("Must provide name for new sheet.")
}
try {
await this.connect()
return await this.client.addSheet({ title: name, headerValues: ["test"] })
return await this.client.addSheet({ title: name, headerValues: [name] })
} catch (err) {
console.error("Error creating new table in google sheets", err)
throw err
}
}
async updateTable(table?: any) {
try {
await this.connect()
const sheet = this.client.sheetsByTitle[table.name]
await sheet.loadHeaderRow()
async updateTable(table: TableRequest) {
await this.connect()
const sheet = this.client.sheetsByTitle[table.name]
await sheet.loadHeaderRow()
if (table._rename) {
const headers = []
for (let header of sheet.headerValues) {
if (header === table._rename.old) {
headers.push(table._rename.updated)
} else {
headers.push(header)
}
if (table._rename) {
const headers = []
for (let header of sheet.headerValues) {
if (header === table._rename.old) {
headers.push(table._rename.updated)
} else {
headers.push(header)
}
await sheet.setHeaderRow(headers)
} else {
const updatedHeaderValues = [...sheet.headerValues]
const newField = Object.keys(table.schema).find(
key => !sheet.headerValues.includes(key)
)
if (newField) {
updatedHeaderValues.push(newField)
}
await sheet.setHeaderRow(updatedHeaderValues)
}
} catch (err) {
console.error("Error updating table in google sheets", err)
throw err
try {
await sheet.setHeaderRow(headers)
} catch (err) {
console.error("Error updating column name in google sheets", err)
throw err
}
} else {
const updatedHeaderValues = [...sheet.headerValues]
// add new column - doesn't currently exist
for (let [key, column] of Object.entries(table.schema)) {
if (!ALLOWED_TYPES.includes(column.type)) {
throw new Error(
`Column type: ${column.type} not allowed for GSheets integration.`
)
}
if (
!sheet.headerValues.includes(key) &&
column.type !== FieldType.FORMULA
) {
updatedHeaderValues.push(key)
}
}
// clear out deleted columns
for (let key of sheet.headerValues) {
if (!Object.keys(table.schema).includes(key)) {
const idx = updatedHeaderValues.indexOf(key)
updatedHeaderValues.splice(idx, 1)
}
}
try {
await sheet.setHeaderRow(updatedHeaderValues)
} catch (err) {
console.error("Error updating table in google sheets", err)
throw err
}
}
}
@ -349,6 +403,24 @@ class GoogleSheetsIntegration implements DatasourcePlus {
}
}
async createBulk(query: { sheet: string; rows: any[] }) {
try {
await this.connect()
const sheet = this.client.sheetsByTitle[query.sheet]
let rowsToInsert = []
for (let row of query.rows) {
rowsToInsert.push(typeof row === "string" ? JSON.parse(row) : row)
}
const rows = await sheet.addRows(rowsToInsert)
return rows.map(row =>
this.buildRowObject(sheet.headerValues, row._rawData, row._rowNumber)
)
} catch (err) {
console.error("Error bulk writing to google sheets", err)
throw err
}
}
async read(query: {
sheet: string
filters?: SearchFilters

20
packages/server/src/integrations/utils.ts
View File

@ -4,6 +4,7 @@ import { FieldTypes, BuildSchemaErrors, InvalidColumns } from "../constants"
const DOUBLE_SEPARATOR = `${SEPARATOR}${SEPARATOR}`
const ROW_ID_REGEX = /^\[.*]$/g
const ENCODED_SPACE = encodeURIComponent(" ")
const SQL_NUMBER_TYPE_MAP = {
integer: FieldTypes.NUMBER,
@ -79,6 +80,10 @@ export function isExternalTable(tableId: string) {
}
export function buildExternalTableId(datasourceId: string, tableName: string) {
// encode spaces
if (tableName.includes(" ")) {
tableName = encodeURIComponent(tableName)
}
return `${datasourceId}${DOUBLE_SEPARATOR}${tableName}`
}
@ -90,6 +95,10 @@ export function breakExternalTableId(tableId: string | undefined) {
let datasourceId = parts.shift()
// if they need joined
let tableName = parts.join(DOUBLE_SEPARATOR)
// if contains encoded spaces, decode it
if (tableName.includes(ENCODED_SPACE)) {
tableName = decodeURIComponent(tableName)
}
return { datasourceId, tableName }
}
@ -200,9 +209,9 @@ export function isIsoDateString(str: string) {
* @param column The column to check, to see if it is a valid relationship.
* @param tableIds The IDs of the tables which currently exist.
*/
function shouldCopyRelationship(
export function shouldCopyRelationship(
column: { type: string; tableId?: string },
tableIds: [string]
tableIds: string[]
) {
return (
column.type === FieldTypes.LINK &&
@ -219,7 +228,7 @@ function shouldCopyRelationship(
* @param column The column to check for options or boolean type.
* @param fetchedColumn The fetched column to check for the type in the external database.
*/
function shouldCopySpecialColumn(
export function shouldCopySpecialColumn(
column: { type: string },
fetchedColumn: { type: string } | undefined
) {
@ -257,9 +266,12 @@ function copyExistingPropsOver(
tableIds: [string]
) {
if (entities && entities[tableName]) {
if (entities[tableName].primaryDisplay) {
if (entities[tableName]?.primaryDisplay) {
table.primaryDisplay = entities[tableName].primaryDisplay
}
if (entities[tableName]?.created) {
table.created = entities[tableName]?.created
}
const existingTableSchema = entities[tableName].schema
for (let key in existingTableSchema) {
if (!existingTableSchema.hasOwnProperty(key)) {

1
packages/types/src/documents/app/table.ts
View File

@ -76,6 +76,7 @@ export interface Table extends Document {
sql?: boolean
indexes?: { [key: string]: any }
rows?: { [key: string]: any }
created?: boolean
}
export interface TableRequest extends Table {