From ffc5360c41d3b03da09f4e24d1c614b97cb4faf4 Mon Sep 17 00:00:00 2001 From: Rory Powell Date: Wed, 29 Mar 2023 08:48:06 +0100 Subject: [PATCH] Revert "Merge pull request #10038 from Budibase/fix/budi-6723" (#10143) This reverts commit 9edd87d9b9085d5ed5f223d65ae21b8372fae587, reversing changes made to 08ec47f9ba5e2f09414411a778fc80446cbe2205. --- .../DataTable/modals/CreateEditColumn.svelte | 9 +- .../CreateExternalTableModal.svelte | 4 +- .../modals/GoogleDatasourceConfigModal.svelte | 32 +-- .../builder/portal/settings/auth/index.svelte | 17 +- .../server/src/api/controllers/datasource.ts | 3 +- .../src/api/controllers/row/external.ts | 42 ++-- .../server/src/api/controllers/row/index.ts | 9 +- .../src/api/controllers/row/internal.ts | 7 + .../server/src/api/controllers/row/utils.ts | 7 +- .../src/api/controllers/table/external.ts | 41 ++-- .../server/src/api/controllers/table/index.ts | 18 +- .../server/src/api/controllers/table/utils.ts | 20 +- packages/server/src/constants/index.ts | 1 - .../server/src/integrations/googlesheets.ts | 210 ++++++------------ packages/server/src/integrations/utils.ts | 20 +- packages/types/src/documents/app/table.ts | 1 - 16 files changed, 145 insertions(+), 296 deletions(-) diff --git a/packages/builder/src/components/backend/DataTable/modals/CreateEditColumn.svelte b/packages/builder/src/components/backend/DataTable/modals/CreateEditColumn.svelte index b4293a2a0a..352f094507 100644 --- a/packages/builder/src/components/backend/DataTable/modals/CreateEditColumn.svelte +++ b/packages/builder/src/components/backend/DataTable/modals/CreateEditColumn.svelte @@ -308,7 +308,7 @@ { name: "Auto Column", type: AUTO_TYPE }, ] } else { - let fields = [ + return [ FIELDS.STRING, FIELDS.BARCODEQR, FIELDS.LONGFORM, @@ -316,13 +316,10 @@ FIELDS.DATETIME, FIELDS.NUMBER, FIELDS.BOOLEAN, + FIELDS.ARRAY, FIELDS.FORMULA, + FIELDS.LINK, ] - // no-sql or a spreadsheet - if (!external || table.sql) { - fields = [...fields, FIELDS.LINK, FIELDS.ARRAY] - } - return fields } } diff --git a/packages/builder/src/components/backend/DatasourceNavigator/TableIntegrationMenu/CreateExternalTableModal.svelte b/packages/builder/src/components/backend/DatasourceNavigator/TableIntegrationMenu/CreateExternalTableModal.svelte index 664b5629d4..45269a365c 100644 --- a/packages/builder/src/components/backend/DatasourceNavigator/TableIntegrationMenu/CreateExternalTableModal.svelte +++ b/packages/builder/src/components/backend/DatasourceNavigator/TableIntegrationMenu/CreateExternalTableModal.svelte @@ -35,9 +35,7 @@ await datasources.fetch() $goto(`../../table/${table._id}`) } catch (error) { - notifications.error( - `Error saving table - ${error?.message || "unknown error"}` - ) + notifications.error("Error saving table") } } diff --git a/packages/builder/src/components/backend/DatasourceNavigator/modals/GoogleDatasourceConfigModal.svelte b/packages/builder/src/components/backend/DatasourceNavigator/modals/GoogleDatasourceConfigModal.svelte index c12ddab78d..7d03dafeb9 100644 --- a/packages/builder/src/components/backend/DatasourceNavigator/modals/GoogleDatasourceConfigModal.svelte +++ b/packages/builder/src/components/backend/DatasourceNavigator/modals/GoogleDatasourceConfigModal.svelte @@ -1,22 +1,15 @@ - - {#if isGoogleConfigured === true} - - Authenticate with your google account to use the {IntegrationNames[ - datasource.type - ]} integration. - - save(datasource, true)} /> - {:else if isGoogleConfigured === false} - Google authentication is not enabled, please complete Google SSO - configuration. + Authenticate with your google account to use the {IntegrationNames[ + datasource.type + ]} integration. - Configure Google SSO - {/if} + + save(datasource, true)} /> diff --git a/packages/builder/src/pages/builder/portal/settings/auth/index.svelte b/packages/builder/src/pages/builder/portal/settings/auth/index.svelte index 2d4dc7ee46..0e82dd31e7 100644 --- a/packages/builder/src/pages/builder/portal/settings/auth/index.svelte +++ b/packages/builder/src/pages/builder/portal/settings/auth/index.svelte @@ -47,9 +47,8 @@ $: googleCallbackTooltip = $admin.cloud ? null : googleCallbackReadonly - ? "Visit the organisation page to update the platform URL" + ? "Vist the organisation page to update the platform URL" : "Leave blank to use the default callback URL" - $: googleSheetsCallbackUrl = `${$organisation.platformUrl}/api/global/auth/datasource/google/callback` $: GoogleConfigFields = { Google: [ @@ -63,14 +62,6 @@ placeholder: $organisation.googleCallbackUrl, copyButton: true, }, - { - name: "sheetsURL", - label: "Sheets URL", - readonly: googleCallbackReadonly, - tooltip: googleCallbackTooltip, - placeholder: googleSheetsCallbackUrl, - copyButton: true, - }, ], } @@ -405,11 +396,7 @@ To allow users to authenticate using their Google accounts, fill out the - fields below. Read the documentation for more information. + fields below. diff --git a/packages/server/src/api/controllers/datasource.ts b/packages/server/src/api/controllers/datasource.ts index 3d41cd89af..d212f7f361 100644 --- a/packages/server/src/api/controllers/datasource.ts +++ b/packages/server/src/api/controllers/datasource.ts @@ -84,9 +84,8 @@ export async function buildSchemaFromDb(ctx: UserCtx) { setDefaultDisplayColumns(datasource) const dbResp = await db.put(datasource) datasource._rev = dbResp.rev - const cleanedDatasource = await sdk.datasources.removeSecretSingle(datasource) - const response: any = { datasource: cleanedDatasource } + const response: any = { datasource } if (error) { response.error = error } diff --git a/packages/server/src/api/controllers/row/external.ts b/packages/server/src/api/controllers/row/external.ts index ee796e32d4..8a7a9a6c69 100644 --- a/packages/server/src/api/controllers/row/external.ts +++ b/packages/server/src/api/controllers/row/external.ts @@ -12,7 +12,7 @@ import * as exporters from "../view/exporters" import { apiFileReturn } from "../../../utilities/fileSystem" import { Operation, - UserCtx, + BBContext, Row, PaginationJson, Table, @@ -21,7 +21,6 @@ import { SortJson, } from "@budibase/types" import sdk from "../../../sdk" -import * as utils from "./utils" const { cleanExportRows } = require("./utils") @@ -50,19 +49,12 @@ export async function handleRequest( ) } -export async function patch(ctx: UserCtx) { +export async function patch(ctx: BBContext) { const inputs = ctx.request.body const tableId = ctx.params.tableId const id = inputs._id // don't save the ID to db delete inputs._id - const validateResult = await utils.validate({ - row: inputs, - tableId, - }) - if (!validateResult.valid) { - throw { validation: validateResult.errors } - } return handleRequest(Operation.UPDATE, tableId, { id: breakRowIdField(id), row: inputs, @@ -70,23 +62,16 @@ export async function patch(ctx: UserCtx) { }) } -export async function save(ctx: UserCtx) { +export async function save(ctx: BBContext) { const inputs = ctx.request.body const tableId = ctx.params.tableId - const validateResult = await utils.validate({ - row: inputs, - tableId, - }) - if (!validateResult.valid) { - throw { validation: validateResult.errors } - } return handleRequest(Operation.CREATE, tableId, { row: inputs, includeSqlRelationships: IncludeRelationship.EXCLUDE, }) } -export async function fetchView(ctx: UserCtx) { +export async function fetchView(ctx: BBContext) { // there are no views in external datasources, shouldn't ever be called // for now just fetch const split = ctx.params.viewName.split("all_") @@ -94,14 +79,14 @@ export async function fetchView(ctx: UserCtx) { return fetch(ctx) } -export async function fetch(ctx: UserCtx) { +export async function fetch(ctx: BBContext) { const tableId = ctx.params.tableId return handleRequest(Operation.READ, tableId, { includeSqlRelationships: IncludeRelationship.INCLUDE, }) } -export async function find(ctx: UserCtx) { +export async function find(ctx: BBContext) { const id = ctx.params.rowId const tableId = ctx.params.tableId const response = (await handleRequest(Operation.READ, tableId, { @@ -111,7 +96,7 @@ export async function find(ctx: UserCtx) { return response ? response[0] : response } -export async function destroy(ctx: UserCtx) { +export async function destroy(ctx: BBContext) { const tableId = ctx.params.tableId const id = ctx.request.body._id const { row } = (await handleRequest(Operation.DELETE, tableId, { @@ -121,7 +106,7 @@ export async function destroy(ctx: UserCtx) { return { response: { ok: true }, row } } -export async function bulkDestroy(ctx: UserCtx) { +export async function bulkDestroy(ctx: BBContext) { const { rows } = ctx.request.body const tableId = ctx.params.tableId let promises: Promise[] = [] @@ -137,7 +122,7 @@ export async function bulkDestroy(ctx: UserCtx) { return { response: { ok: true }, rows: responses.map(resp => resp.row) } } -export async function search(ctx: UserCtx) { +export async function search(ctx: BBContext) { const tableId = ctx.params.tableId const { paginate, query, ...params } = ctx.request.body let { bookmark, limit } = params @@ -200,7 +185,12 @@ export async function search(ctx: UserCtx) { } } -export async function exportRows(ctx: UserCtx) { +export async function validate(ctx: BBContext) { + // can't validate external right now - maybe in future + return { valid: true } +} + +export async function exportRows(ctx: BBContext) { const { datasourceId, tableName } = breakExternalTableId(ctx.params.tableId) const format = ctx.query.format const { columns } = ctx.request.body @@ -254,7 +244,7 @@ export async function exportRows(ctx: UserCtx) { return apiFileReturn(exporter(headers, exportRows)) } -export async function fetchEnrichedRow(ctx: UserCtx) { +export async function fetchEnrichedRow(ctx: BBContext) { const id = ctx.params.rowId const tableId = ctx.params.tableId const { datasourceId, tableName } = breakExternalTableId(tableId) diff --git a/packages/server/src/api/controllers/row/index.ts b/packages/server/src/api/controllers/row/index.ts index 348d441c78..b59f245098 100644 --- a/packages/server/src/api/controllers/row/index.ts +++ b/packages/server/src/api/controllers/row/index.ts @@ -2,8 +2,6 @@ import { quotas } from "@budibase/pro" import * as internal from "./internal" import * as external from "./external" import { isExternalTable } from "../../../integrations/utils" -import { Ctx } from "@budibase/types" -import * as utils from "./utils" function pickApi(tableId: any) { if (isExternalTable(tableId)) { @@ -131,12 +129,9 @@ export async function search(ctx: any) { }) } -export async function validate(ctx: Ctx) { +export async function validate(ctx: any) { const tableId = getTableId(ctx) - ctx.body = await utils.validate({ - row: ctx.request.body, - tableId, - }) + ctx.body = await pickApi(tableId).validate(ctx) } export async function fetchEnrichedRow(ctx: any) { diff --git a/packages/server/src/api/controllers/row/internal.ts b/packages/server/src/api/controllers/row/internal.ts index 039f03c015..d36f9bf2f1 100644 --- a/packages/server/src/api/controllers/row/internal.ts +++ b/packages/server/src/api/controllers/row/internal.ts @@ -387,6 +387,13 @@ export async function search(ctx: Ctx) { return response } +export async function validate(ctx: Ctx) { + return utils.validate({ + tableId: ctx.params.tableId, + row: ctx.request.body, + }) +} + export async function exportRows(ctx: Ctx) { const db = context.getAppDB() const table = await db.get(ctx.params.tableId) diff --git a/packages/server/src/api/controllers/row/utils.ts b/packages/server/src/api/controllers/row/utils.ts index 2e8f2f4536..82232b7f98 100644 --- a/packages/server/src/api/controllers/row/utils.ts +++ b/packages/server/src/api/controllers/row/utils.ts @@ -4,11 +4,11 @@ import { FieldTypes } from "../../../constants" import { context } from "@budibase/backend-core" import { makeExternalQuery } from "../../../integrations/base/query" import { Row, Table } from "@budibase/types" +const validateJs = require("validate.js") +const { cloneDeep } = require("lodash/fp") import { Format } from "../view/exporters" import { Ctx } from "@budibase/types" import sdk from "../../../sdk" -const validateJs = require("validate.js") -const { cloneDeep } = require("lodash/fp") validateJs.extend(validateJs.validators.datetime, { parse: function (value: string) { @@ -56,7 +56,8 @@ export async function validate({ }) { let fetchedTable: Table if (!table) { - fetchedTable = await sdk.tables.getTable(tableId) + const db = context.getAppDB() + fetchedTable = await db.get(tableId) } else { fetchedTable = table } diff --git a/packages/server/src/api/controllers/table/external.ts b/packages/server/src/api/controllers/table/external.ts index a00e65687f..a6ad5bba99 100644 --- a/packages/server/src/api/controllers/table/external.ts +++ b/packages/server/src/api/controllers/table/external.ts @@ -7,7 +7,6 @@ import { generateJunctionTableName, foreignKeyStructure, hasTypeChanged, - setStaticSchemas, } from "./utils" import { FieldTypes } from "../../../constants" import { makeExternalQuery } from "../../../integrations/base/query" @@ -21,7 +20,7 @@ import { Operation, RenameColumn, FieldSchema, - UserCtx, + BBContext, TableRequest, RelationshipTypes, } from "@budibase/types" @@ -195,20 +194,20 @@ function isRelationshipSetup(column: FieldSchema) { return column.foreignKey || column.through } -export async function save(ctx: UserCtx) { - const inputs: TableRequest = ctx.request.body - const renamed = inputs?._rename +export async function save(ctx: BBContext) { + const table: TableRequest = ctx.request.body + const renamed = table?._rename // can't do this right now - delete inputs.rows + delete table.rows const datasourceId = getDatasourceId(ctx.request.body)! // table doesn't exist already, note that it is created - if (!inputs._id) { - inputs.created = true + if (!table._id) { + table.created = true } let tableToSave: TableRequest = { type: "table", - _id: buildExternalTableId(datasourceId, inputs.name), - ...inputs, + _id: buildExternalTableId(datasourceId, table.name), + ...table, } let oldTable @@ -225,10 +224,6 @@ export async function save(ctx: UserCtx) { if (!datasource.entities) { datasource.entities = {} } - - // GSheets is a specific case - only ever has a static primary key - tableToSave = setStaticSchemas(datasource, tableToSave) - const oldTables = cloneDeep(datasource.entities) const tables: Record = datasource.entities @@ -251,7 +246,7 @@ export async function save(ctx: UserCtx) { const junctionTable = generateManyLinkSchema( datasource, schema, - tableToSave, + table, relatedTable ) if (tables[junctionTable.name]) { @@ -261,12 +256,10 @@ export async function save(ctx: UserCtx) { extraTablesToUpdate.push(junctionTable) } else { const fkTable = - relationType === RelationshipTypes.ONE_TO_MANY - ? tableToSave - : relatedTable + relationType === RelationshipTypes.ONE_TO_MANY ? table : relatedTable const foreignKey = generateLinkSchema( schema, - tableToSave, + table, relatedTable, relationType ) @@ -278,11 +271,11 @@ export async function save(ctx: UserCtx) { fkTable.constrained.push(foreignKey) } // foreign key is in other table, need to save it to external - if (fkTable._id !== tableToSave._id) { + if (fkTable._id !== table._id) { extraTablesToUpdate.push(fkTable) } } - generateRelatedSchema(schema, relatedTable, tableToSave, relatedColumnName) + generateRelatedSchema(schema, relatedTable, table, relatedColumnName) schema.main = true } @@ -320,7 +313,7 @@ export async function save(ctx: UserCtx) { return tableToSave } -export async function destroy(ctx: UserCtx) { +export async function destroy(ctx: BBContext) { const tableToDelete: TableRequest = await sdk.tables.getTable( ctx.params.tableId ) @@ -346,7 +339,7 @@ export async function destroy(ctx: UserCtx) { return tableToDelete } -export async function bulkImport(ctx: UserCtx) { +export async function bulkImport(ctx: BBContext) { const table = await sdk.tables.getTable(ctx.params.tableId) const { rows }: { rows: unknown } = ctx.request.body const schema: unknown = table.schema @@ -355,7 +348,7 @@ export async function bulkImport(ctx: UserCtx) { ctx.throw(400, "Provided data import information is invalid.") } - const parsedRows = parse(rows, schema) + const parsedRows = await parse(rows, schema) await handleRequest(Operation.BULK_CREATE, table._id!, { rows: parsedRows, }) diff --git a/packages/server/src/api/controllers/table/index.ts b/packages/server/src/api/controllers/table/index.ts index 2ab7ad7b38..aa6dfde536 100644 --- a/packages/server/src/api/controllers/table/index.ts +++ b/packages/server/src/api/controllers/table/index.ts @@ -8,7 +8,7 @@ import { import { isExternalTable, isSQL } from "../../../integrations/utils" import { getDatasourceParams } from "../../../db/utils" import { context, events } from "@budibase/backend-core" -import { Table, UserCtx } from "@budibase/types" +import { Table, BBContext } from "@budibase/types" import sdk from "../../../sdk" import csv from "csvtojson" @@ -25,7 +25,7 @@ function pickApi({ tableId, table }: { tableId?: string; table?: Table }) { } // covers both internal and external -export async function fetch(ctx: UserCtx) { +export async function fetch(ctx: BBContext) { const db = context.getAppDB() const internal = await sdk.tables.getAllInternalTables() @@ -53,12 +53,12 @@ export async function fetch(ctx: UserCtx) { ctx.body = [...internal, ...external] } -export async function find(ctx: UserCtx) { +export async function find(ctx: BBContext) { const tableId = ctx.params.tableId ctx.body = await sdk.tables.getTable(tableId) } -export async function save(ctx: UserCtx) { +export async function save(ctx: BBContext) { const appId = ctx.appId const table = ctx.request.body const isImport = table.rows @@ -79,7 +79,7 @@ export async function save(ctx: UserCtx) { ctx.body = savedTable } -export async function destroy(ctx: UserCtx) { +export async function destroy(ctx: BBContext) { const appId = ctx.appId const tableId = ctx.params.tableId const deletedTable = await pickApi({ tableId }).destroy(ctx) @@ -91,7 +91,7 @@ export async function destroy(ctx: UserCtx) { ctx.body = { message: `Table ${tableId} deleted.` } } -export async function bulkImport(ctx: UserCtx) { +export async function bulkImport(ctx: BBContext) { const tableId = ctx.params.tableId await pickApi({ tableId }).bulkImport(ctx) // right now we don't trigger anything for bulk import because it @@ -101,7 +101,7 @@ export async function bulkImport(ctx: UserCtx) { ctx.body = { message: `Bulk rows created.` } } -export async function csvToJson(ctx: UserCtx) { +export async function csvToJson(ctx: BBContext) { const { csvString } = ctx.request.body const result = await csv().fromString(csvString) @@ -110,7 +110,7 @@ export async function csvToJson(ctx: UserCtx) { ctx.body = result } -export async function validateNewTableImport(ctx: UserCtx) { +export async function validateNewTableImport(ctx: BBContext) { const { rows, schema }: { rows: unknown; schema: unknown } = ctx.request.body if (isRows(rows) && isSchema(schema)) { @@ -121,7 +121,7 @@ export async function validateNewTableImport(ctx: UserCtx) { } } -export async function validateExistingTableImport(ctx: UserCtx) { +export async function validateExistingTableImport(ctx: BBContext) { const { rows, tableId }: { rows: unknown; tableId: unknown } = ctx.request.body diff --git a/packages/server/src/api/controllers/table/utils.ts b/packages/server/src/api/controllers/table/utils.ts index 7c5c81939a..bbccde467b 100644 --- a/packages/server/src/api/controllers/table/utils.ts +++ b/packages/server/src/api/controllers/table/utils.ts @@ -1,11 +1,7 @@ import { parse, isSchema, isRows } from "../../../utilities/schema" import { getRowParams, generateRowID, InternalTables } from "../../../db/utils" import { isEqual } from "lodash" -import { - AutoFieldSubTypes, - FieldTypes, - GOOGLE_SHEETS_PRIMARY_KEY, -} from "../../../constants" +import { AutoFieldSubTypes, FieldTypes } from "../../../constants" import { inputProcessing, cleanupAttachments, @@ -20,7 +16,7 @@ import viewTemplate from "../view/viewBuilder" import { cloneDeep } from "lodash/fp" import { quotas } from "@budibase/pro" import { events, context } from "@budibase/backend-core" -import { Database, Datasource, SourceName, Table } from "@budibase/types" +import { Database } from "@budibase/types" export async function clearColumns(table: any, columnNames: any) { const db: Database = context.getAppDB() @@ -396,17 +392,5 @@ export function hasTypeChanged(table: any, oldTable: any) { return false } -// used for external tables, some of them will have static schemas that need -// to be hard set -export function setStaticSchemas(datasource: Datasource, table: Table) { - // GSheets is a specific case - only ever has a static primary key - if (table && datasource.source === SourceName.GOOGLE_SHEETS) { - table.primary = [GOOGLE_SHEETS_PRIMARY_KEY] - // if there is an id column, remove it, should never exist in GSheets - delete table.schema?.id - } - return table -} - const _TableSaveFunctions = TableSaveFunctions export { _TableSaveFunctions as TableSaveFunctions } diff --git a/packages/server/src/constants/index.ts b/packages/server/src/constants/index.ts index 9d6a1c247a..e55ad09add 100644 --- a/packages/server/src/constants/index.ts +++ b/packages/server/src/constants/index.ts @@ -180,4 +180,3 @@ export enum AutomationErrors { // pass through the list from the auth/core lib export const ObjectStoreBuckets = objectStore.ObjectStoreBuckets export const MAX_AUTOMATION_RECURRING_ERRORS = 5 -export const GOOGLE_SHEETS_PRIMARY_KEY = "rowNumber" diff --git a/packages/server/src/integrations/googlesheets.ts b/packages/server/src/integrations/googlesheets.ts index f8bc84adea..0c658df0f5 100644 --- a/packages/server/src/integrations/googlesheets.ts +++ b/packages/server/src/integrations/googlesheets.ts @@ -1,25 +1,22 @@ import { DatasourceFieldType, DatasourcePlus, - FieldType, Integration, - Operation, PaginationJson, QueryJson, QueryType, - Row, SearchFilters, SortJson, Table, - TableRequest, + TableSchema, } from "@budibase/types" import { OAuth2Client } from "google-auth-library" -import { buildExternalTableId, finaliseExternalTables } from "./utils" +import { buildExternalTableId } from "./utils" +import { DataSourceOperation, FieldTypes } from "../constants" import { GoogleSpreadsheet } from "google-spreadsheet" import fetch from "node-fetch" import { configs, HTTPError } from "@budibase/backend-core" import { dataFilters } from "@budibase/shared-core" -import { GOOGLE_SHEETS_PRIMARY_KEY } from "../constants" interface GoogleSheetsConfig { spreadsheetId: string @@ -42,17 +39,6 @@ interface AuthTokenResponse { access_token: string } -const ALLOWED_TYPES = [ - FieldType.STRING, - FieldType.FORMULA, - FieldType.NUMBER, - FieldType.LONGFORM, - FieldType.DATETIME, - FieldType.OPTIONS, - FieldType.BOOLEAN, - FieldType.BARCODEQR, -] - const SCHEMA: Integration = { plus: true, auth: { @@ -213,90 +199,73 @@ class GoogleSheetsIntegration implements DatasourcePlus { this.client.useOAuth2Client(oauthClient) await this.client.loadInfo() - } catch (err: any) { - // this happens for xlsx imports - if (err.message?.includes("operation is not supported")) { - err.message = - "This operation is not supported - XLSX sheets must be converted." - } + } catch (err) { console.error("Error connecting to google sheets", err) throw err } } - getTableSchema(title: string, headerValues: string[], id?: string) { - // base table - const table: Table = { - name: title, - primary: [GOOGLE_SHEETS_PRIMARY_KEY], - schema: {}, - } - if (id) { - table._id = id - } - // build schema from headers - for (let header of headerValues) { - table.schema[header] = { - name: header, - type: FieldType.STRING, - } - } - return table - } - - async buildSchema(datasourceId: string, entities: Record) { + async buildSchema(datasourceId: string) { await this.connect() const sheets = this.client.sheetsByIndex const tables: Record = {} for (let sheet of sheets) { // must fetch rows to determine schema await sheet.getRows() + // build schema + const schema: TableSchema = {} - const id = buildExternalTableId(datasourceId, sheet.title) - tables[sheet.title] = this.getTableSchema( - sheet.title, - sheet.headerValues, - id - ) + // build schema from headers + for (let header of sheet.headerValues) { + schema[header] = { + name: header, + type: FieldTypes.STRING, + } + } + + // create tables + tables[sheet.title] = { + _id: buildExternalTableId(datasourceId, sheet.title), + name: sheet.title, + primary: ["rowNumber"], + schema, + } } - const final = finaliseExternalTables(tables, entities) - this.tables = final.tables - this.schemaErrors = final.errors + + this.tables = tables } async query(json: QueryJson) { const sheet = json.endpoint.entityId - switch (json.endpoint.operation) { - case Operation.CREATE: - return this.create({ sheet, row: json.body as Row }) - case Operation.BULK_CREATE: - return this.createBulk({ sheet, rows: json.body as Row[] }) - case Operation.READ: - return this.read({ ...json, sheet }) - case Operation.UPDATE: - return this.update({ + + const handlers = { + [DataSourceOperation.CREATE]: () => + this.create({ sheet, row: json.body }), + [DataSourceOperation.READ]: () => this.read({ ...json, sheet }), + [DataSourceOperation.UPDATE]: () => + this.update({ // exclude the header row and zero index rowIndex: json.extra?.idFilter?.equal?.rowNumber - 2, sheet, row: json.body, - }) - case Operation.DELETE: - return this.delete({ + }), + [DataSourceOperation.DELETE]: () => + this.delete({ // exclude the header row and zero index rowIndex: json.extra?.idFilter?.equal?.rowNumber - 2, sheet, - }) - case Operation.CREATE_TABLE: - return this.createTable(json?.table?.name) - case Operation.UPDATE_TABLE: - return this.updateTable(json.table!) - case Operation.DELETE_TABLE: - return this.deleteTable(json?.table?.name) - default: - throw new Error( - `GSheets integration does not support "${json.endpoint.operation}".` - ) + }), + [DataSourceOperation.CREATE_TABLE]: () => + this.createTable(json?.table?.name), + [DataSourceOperation.UPDATE_TABLE]: () => this.updateTable(json.table), + [DataSourceOperation.DELETE_TABLE]: () => + this.deleteTable(json?.table?.name), } + + // @ts-ignore + const internalQueryMethod = handlers[json.endpoint.operation] + + return await internalQueryMethod() } buildRowObject(headers: string[], values: string[], rowNumber: number) { @@ -309,70 +278,47 @@ class GoogleSheetsIntegration implements DatasourcePlus { } async createTable(name?: string) { - if (!name) { - throw new Error("Must provide name for new sheet.") - } try { await this.connect() - return await this.client.addSheet({ title: name, headerValues: [name] }) + return await this.client.addSheet({ title: name, headerValues: ["test"] }) } catch (err) { console.error("Error creating new table in google sheets", err) throw err } } - async updateTable(table: TableRequest) { - await this.connect() - const sheet = this.client.sheetsByTitle[table.name] - await sheet.loadHeaderRow() + async updateTable(table?: any) { + try { + await this.connect() + const sheet = this.client.sheetsByTitle[table.name] + await sheet.loadHeaderRow() - if (table._rename) { - const headers = [] - for (let header of sheet.headerValues) { - if (header === table._rename.old) { - headers.push(table._rename.updated) - } else { - headers.push(header) + if (table._rename) { + const headers = [] + for (let header of sheet.headerValues) { + if (header === table._rename.old) { + headers.push(table._rename.updated) + } else { + headers.push(header) + } } - } - try { await sheet.setHeaderRow(headers) - } catch (err) { - console.error("Error updating column name in google sheets", err) - throw err - } - } else { - const updatedHeaderValues = [...sheet.headerValues] + } else { + const updatedHeaderValues = [...sheet.headerValues] - // add new column - doesn't currently exist - for (let [key, column] of Object.entries(table.schema)) { - if (!ALLOWED_TYPES.includes(column.type)) { - throw new Error( - `Column type: ${column.type} not allowed for GSheets integration.` - ) - } - if ( - !sheet.headerValues.includes(key) && - column.type !== FieldType.FORMULA - ) { - updatedHeaderValues.push(key) - } - } + const newField = Object.keys(table.schema).find( + key => !sheet.headerValues.includes(key) + ) - // clear out deleted columns - for (let key of sheet.headerValues) { - if (!Object.keys(table.schema).includes(key)) { - const idx = updatedHeaderValues.indexOf(key) - updatedHeaderValues.splice(idx, 1) + if (newField) { + updatedHeaderValues.push(newField) } - } - try { await sheet.setHeaderRow(updatedHeaderValues) - } catch (err) { - console.error("Error updating table in google sheets", err) - throw err } + } catch (err) { + console.error("Error updating table in google sheets", err) + throw err } } @@ -403,24 +349,6 @@ class GoogleSheetsIntegration implements DatasourcePlus { } } - async createBulk(query: { sheet: string; rows: any[] }) { - try { - await this.connect() - const sheet = this.client.sheetsByTitle[query.sheet] - let rowsToInsert = [] - for (let row of query.rows) { - rowsToInsert.push(typeof row === "string" ? JSON.parse(row) : row) - } - const rows = await sheet.addRows(rowsToInsert) - return rows.map(row => - this.buildRowObject(sheet.headerValues, row._rawData, row._rowNumber) - ) - } catch (err) { - console.error("Error bulk writing to google sheets", err) - throw err - } - } - async read(query: { sheet: string filters?: SearchFilters diff --git a/packages/server/src/integrations/utils.ts b/packages/server/src/integrations/utils.ts index de8b318bb1..356a08f4a0 100644 --- a/packages/server/src/integrations/utils.ts +++ b/packages/server/src/integrations/utils.ts @@ -4,7 +4,6 @@ import { FieldTypes, BuildSchemaErrors, InvalidColumns } from "../constants" const DOUBLE_SEPARATOR = `${SEPARATOR}${SEPARATOR}` const ROW_ID_REGEX = /^\[.*]$/g -const ENCODED_SPACE = encodeURIComponent(" ") const SQL_NUMBER_TYPE_MAP = { integer: FieldTypes.NUMBER, @@ -80,10 +79,6 @@ export function isExternalTable(tableId: string) { } export function buildExternalTableId(datasourceId: string, tableName: string) { - // encode spaces - if (tableName.includes(" ")) { - tableName = encodeURIComponent(tableName) - } return `${datasourceId}${DOUBLE_SEPARATOR}${tableName}` } @@ -95,10 +90,6 @@ export function breakExternalTableId(tableId: string | undefined) { let datasourceId = parts.shift() // if they need joined let tableName = parts.join(DOUBLE_SEPARATOR) - // if contains encoded spaces, decode it - if (tableName.includes(ENCODED_SPACE)) { - tableName = decodeURIComponent(tableName) - } return { datasourceId, tableName } } @@ -209,9 +200,9 @@ export function isIsoDateString(str: string) { * @param column The column to check, to see if it is a valid relationship. * @param tableIds The IDs of the tables which currently exist. */ -export function shouldCopyRelationship( +function shouldCopyRelationship( column: { type: string; tableId?: string }, - tableIds: string[] + tableIds: [string] ) { return ( column.type === FieldTypes.LINK && @@ -228,7 +219,7 @@ export function shouldCopyRelationship( * @param column The column to check for options or boolean type. * @param fetchedColumn The fetched column to check for the type in the external database. */ -export function shouldCopySpecialColumn( +function shouldCopySpecialColumn( column: { type: string }, fetchedColumn: { type: string } | undefined ) { @@ -266,12 +257,9 @@ function copyExistingPropsOver( tableIds: [string] ) { if (entities && entities[tableName]) { - if (entities[tableName]?.primaryDisplay) { + if (entities[tableName].primaryDisplay) { table.primaryDisplay = entities[tableName].primaryDisplay } - if (entities[tableName]?.created) { - table.created = entities[tableName]?.created - } const existingTableSchema = entities[tableName].schema for (let key in existingTableSchema) { if (!existingTableSchema.hasOwnProperty(key)) { diff --git a/packages/types/src/documents/app/table.ts b/packages/types/src/documents/app/table.ts index 929409d0e9..01d2486dcb 100644 --- a/packages/types/src/documents/app/table.ts +++ b/packages/types/src/documents/app/table.ts @@ -76,7 +76,6 @@ export interface Table extends Document { sql?: boolean indexes?: { [key: string]: any } rows?: { [key: string]: any } - created?: boolean } export interface TableRequest extends Table {