Merge pull request #13509 from Budibase/feat/budi-8126

[Fix] Datasource plus schema updates issues
This commit is contained in:
Adria Navarro 2024-04-18 08:42:10 +02:00 committed by GitHub
commit e89c5ad112
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
15 changed files with 98 additions and 164 deletions

View File

@ -13,6 +13,7 @@
Layout, Layout,
AbsTooltip, AbsTooltip,
} from "@budibase/bbui" } from "@budibase/bbui"
import { SWITCHABLE_TYPES, ValidColumnNameRegex } from "@budibase/shared-core"
import { createEventDispatcher, getContext, onMount } from "svelte" import { createEventDispatcher, getContext, onMount } from "svelte"
import { cloneDeep } from "lodash/fp" import { cloneDeep } from "lodash/fp"
import { tables, datasources } from "stores/builder" import { tables, datasources } from "stores/builder"
@ -20,11 +21,6 @@
import { import {
FIELDS, FIELDS,
RelationshipType, RelationshipType,
ALLOWABLE_STRING_OPTIONS,
ALLOWABLE_NUMBER_OPTIONS,
ALLOWABLE_STRING_TYPES,
ALLOWABLE_NUMBER_TYPES,
SWITCHABLE_TYPES,
PrettyRelationshipDefinitions, PrettyRelationshipDefinitions,
DB_TYPE_EXTERNAL, DB_TYPE_EXTERNAL,
} from "constants/backend" } from "constants/backend"
@ -33,7 +29,6 @@
import ModalBindableInput from "components/common/bindings/ModalBindableInput.svelte" import ModalBindableInput from "components/common/bindings/ModalBindableInput.svelte"
import { getBindings } from "components/backend/DataTable/formula" import { getBindings } from "components/backend/DataTable/formula"
import JSONSchemaModal from "./JSONSchemaModal.svelte" import JSONSchemaModal from "./JSONSchemaModal.svelte"
import { ValidColumnNameRegex } from "@budibase/shared-core"
import { FieldType, FieldSubtype, SourceName } from "@budibase/types" import { FieldType, FieldSubtype, SourceName } from "@budibase/types"
import RelationshipSelector from "components/common/RelationshipSelector.svelte" import RelationshipSelector from "components/common/RelationshipSelector.svelte"
import { RowUtils } from "@budibase/frontend-core" import { RowUtils } from "@budibase/frontend-core"
@ -61,8 +56,8 @@
let primaryDisplay let primaryDisplay
let indexes = [...($tables.selected.indexes || [])] let indexes = [...($tables.selected.indexes || [])]
let isCreating = undefined let isCreating = undefined
let relationshipPart1 = PrettyRelationshipDefinitions.Many let relationshipPart1 = PrettyRelationshipDefinitions.MANY
let relationshipPart2 = PrettyRelationshipDefinitions.One let relationshipPart2 = PrettyRelationshipDefinitions.ONE
let relationshipTableIdPrimary = null let relationshipTableIdPrimary = null
let relationshipTableIdSecondary = null let relationshipTableIdSecondary = null
let table = $tables.selected let table = $tables.selected
@ -175,7 +170,7 @@
$: typeEnabled = $: typeEnabled =
!originalName || !originalName ||
(originalName && (originalName &&
SWITCHABLE_TYPES.indexOf(editableColumn.type) !== -1 && SWITCHABLE_TYPES[field.type] &&
!editableColumn?.autocolumn) !editableColumn?.autocolumn)
const fieldDefinitions = Object.values(FIELDS).reduce( const fieldDefinitions = Object.values(FIELDS).reduce(
@ -367,16 +362,15 @@
} }
function getAllowedTypes() { function getAllowedTypes() {
if ( if (originalName) {
originalName && const possibleTypes = (
ALLOWABLE_STRING_TYPES.indexOf(editableColumn.type) !== -1 SWITCHABLE_TYPES[field.type] || [editableColumn.type]
) { ).map(t => t.toLowerCase())
return ALLOWABLE_STRING_OPTIONS return Object.entries(FIELDS)
} else if ( .filter(([fieldType]) =>
originalName && possibleTypes.includes(fieldType.toLowerCase())
ALLOWABLE_NUMBER_TYPES.indexOf(editableColumn.type) !== -1 )
) { .map(([_, fieldDefinition]) => fieldDefinition)
return ALLOWABLE_NUMBER_OPTIONS
} }
const isUsers = const isUsers =

View File

@ -202,26 +202,6 @@ export const PrettyRelationshipDefinitions = {
ONE: "One row", ONE: "One row",
} }
export const ALLOWABLE_STRING_OPTIONS = [
FIELDS.STRING,
FIELDS.OPTIONS,
FIELDS.LONGFORM,
FIELDS.BARCODEQR,
]
export const ALLOWABLE_STRING_TYPES = ALLOWABLE_STRING_OPTIONS.map(
opt => opt.type
)
export const ALLOWABLE_NUMBER_OPTIONS = [FIELDS.NUMBER, FIELDS.BOOLEAN]
export const ALLOWABLE_NUMBER_TYPES = ALLOWABLE_NUMBER_OPTIONS.map(
opt => opt.type
)
export const SWITCHABLE_TYPES = [
...ALLOWABLE_STRING_TYPES,
...ALLOWABLE_NUMBER_TYPES,
]
export const BUDIBASE_INTERNAL_DB_ID = INTERNAL_TABLE_SOURCE_ID export const BUDIBASE_INTERNAL_DB_ID = INTERNAL_TABLE_SOURCE_ID
export const DEFAULT_BB_DATASOURCE_ID = "datasource_internal_bb_default" export const DEFAULT_BB_DATASOURCE_ID = "datasource_internal_bb_default"
export const BUDIBASE_DATASOURCE_TYPE = "budibase" export const BUDIBASE_DATASOURCE_TYPE = "budibase"

View File

@ -1,8 +1,8 @@
import { FieldType } from "@budibase/types" import { FieldType } from "@budibase/types"
import { SWITCHABLE_TYPES } from "@budibase/shared-core"
import { get, writable, derived } from "svelte/store" import { get, writable, derived } from "svelte/store"
import { cloneDeep } from "lodash/fp" import { cloneDeep } from "lodash/fp"
import { API } from "api" import { API } from "api"
import { SWITCHABLE_TYPES } from "constants/backend"
export function createTablesStore() { export function createTablesStore() {
const store = writable({ const store = writable({
@ -64,7 +64,7 @@ export function createTablesStore() {
if ( if (
oldField != null && oldField != null &&
oldField?.type !== field.type && oldField?.type !== field.type &&
SWITCHABLE_TYPES.indexOf(oldField?.type) === -1 !SWITCHABLE_TYPES[oldField?.type]?.includes(field.type)
) { ) {
updatedTable.schema[key] = oldField updatedTable.schema[key] = oldField
} }
@ -148,12 +148,6 @@ export function createTablesStore() {
if (indexes) { if (indexes) {
draft.indexes = indexes draft.indexes = indexes
} }
// Add object to indicate if column is being added
if (draft.schema[field.name] === undefined) {
draft._add = {
name: field.name,
}
}
draft.schema = { draft.schema = {
...draft.schema, ...draft.schema,
[field.name]: cloneDeep(field), [field.name]: cloneDeep(field),

View File

@ -31,7 +31,6 @@ export async function save(
renaming?: RenameColumn renaming?: RenameColumn
) { ) {
const inputs = ctx.request.body const inputs = ctx.request.body
const adding = inputs?._add
// can't do this right now // can't do this right now
delete inputs.rows delete inputs.rows
const tableId = ctx.request.body._id const tableId = ctx.request.body._id
@ -44,7 +43,7 @@ export async function save(
const { datasource, table } = await sdk.tables.external.save( const { datasource, table } = await sdk.tables.external.save(
datasourceId!, datasourceId!,
inputs, inputs,
{ tableId, renaming, adding } { tableId, renaming }
) )
builderSocket?.emitDatasourceUpdate(ctx, datasource) builderSocket?.emitDatasourceUpdate(ctx, datasource)
return table return table

View File

@ -77,11 +77,6 @@ export async function save(ctx: UserCtx<SaveTableRequest, SaveTableResponse>) {
const renaming = ctx.request.body._rename const renaming = ctx.request.body._rename
const api = pickApi({ table }) const api = pickApi({ table })
// do not pass _rename or _add if saving to CouchDB
if (api === internal) {
delete ctx.request.body._add
delete ctx.request.body._rename
}
let savedTable = await api.save(ctx, renaming) let savedTable = await api.save(ctx, renaming)
if (!table._id) { if (!table._id) {
savedTable = sdk.tables.enrichViewSchemas(savedTable) savedTable = sdk.tables.enrichViewSchemas(savedTable)

View File

@ -16,7 +16,7 @@ export async function save(
ctx: UserCtx<SaveTableRequest, SaveTableResponse>, ctx: UserCtx<SaveTableRequest, SaveTableResponse>,
renaming?: RenameColumn renaming?: RenameColumn
) { ) {
const { rows, ...rest } = ctx.request.body const { _rename, rows, ...rest } = ctx.request.body
let tableToSave: Table = { let tableToSave: Table = {
_id: generateTableID(), _id: generateTableID(),
...rest, ...rest,

View File

@ -219,9 +219,6 @@ describe.each([
it("should add a new column for an internal DB table", async () => { it("should add a new column for an internal DB table", async () => {
const saveTableRequest: SaveTableRequest = { const saveTableRequest: SaveTableRequest = {
_add: {
name: "NEW_COLUMN",
},
...basicTable(), ...basicTable(),
} }
@ -235,7 +232,6 @@ describe.each([
updatedAt: expect.stringMatching(ISO_REGEX_PATTERN), updatedAt: expect.stringMatching(ISO_REGEX_PATTERN),
views: {}, views: {},
} }
delete expectedResponse._add
expect(response).toEqual(expectedResponse) expect(response).toEqual(expectedResponse)
}) })
}) })

View File

@ -16,7 +16,6 @@ import {
getDatasource, getDatasource,
rawQuery, rawQuery,
} from "../integrations/tests/utils" } from "../integrations/tests/utils"
import { builderSocket } from "../websockets"
import { generator } from "@budibase/backend-core/tests" import { generator } from "@budibase/backend-core/tests"
// @ts-ignore // @ts-ignore
fetch.mockSearch() fetch.mockSearch()
@ -233,72 +232,6 @@ describe("mysql integrations", () => {
}) })
describe("POST /api/tables/", () => { describe("POST /api/tables/", () => {
const emitDatasourceUpdateMock = jest.fn()
it("will emit the datasource entity schema with externalType to the front-end when adding a new column", async () => {
const addColumnToTable: TableRequest = {
type: "table",
sourceType: TableSourceType.EXTERNAL,
name: uniqueTableName(),
sourceId: datasource._id!,
primary: ["id"],
schema: {
id: {
type: FieldType.AUTO,
name: "id",
autocolumn: true,
},
new_column: {
type: FieldType.NUMBER,
name: "new_column",
},
},
_add: {
name: "new_column",
},
}
jest
.spyOn(builderSocket!, "emitDatasourceUpdate")
.mockImplementation(emitDatasourceUpdateMock)
await makeRequest("post", "/api/tables/", addColumnToTable)
const expectedTable: TableRequest = {
...addColumnToTable,
schema: {
id: {
type: FieldType.NUMBER,
name: "id",
autocolumn: true,
constraints: {
presence: false,
},
externalType: "int unsigned",
},
new_column: {
type: FieldType.NUMBER,
name: "new_column",
autocolumn: false,
constraints: {
presence: false,
},
externalType: "float(8,2)",
},
},
created: true,
_id: `${datasource._id}__${addColumnToTable.name}`,
}
delete expectedTable._add
expect(emitDatasourceUpdateMock).toHaveBeenCalledTimes(1)
const emittedDatasource: Datasource =
emitDatasourceUpdateMock.mock.calls[0][1]
expect(emittedDatasource.entities![expectedTable.name]).toEqual(
expectedTable
)
})
it("will rename a column", async () => { it("will rename a column", async () => {
await makeRequest("post", "/api/tables/", primaryMySqlTable) await makeRequest("post", "/api/tables/", primaryMySqlTable)

View File

@ -7,7 +7,7 @@ import {
} from "@budibase/types" } from "@budibase/types"
import { DocumentType, SEPARATOR } from "../db/utils" import { DocumentType, SEPARATOR } from "../db/utils"
import { InvalidColumns, DEFAULT_BB_DATASOURCE_ID } from "../constants" import { InvalidColumns, DEFAULT_BB_DATASOURCE_ID } from "../constants"
import { helpers } from "@budibase/shared-core" import { SWITCHABLE_TYPES, helpers } from "@budibase/shared-core"
import env from "../environment" import env from "../environment"
import { Knex } from "knex" import { Knex } from "knex"
@ -284,8 +284,8 @@ export function isIsoDateString(str: string) {
* @param column The column to check, to see if it is a valid relationship. * @param column The column to check, to see if it is a valid relationship.
* @param tableIds The IDs of the tables which currently exist. * @param tableIds The IDs of the tables which currently exist.
*/ */
export function shouldCopyRelationship( function shouldCopyRelationship(
column: { type: string; tableId?: string }, column: { type: FieldType.LINK; tableId?: string },
tableIds: string[] tableIds: string[]
) { ) {
return ( return (
@ -303,28 +303,18 @@ export function shouldCopyRelationship(
* @param column The column to check for options or boolean type. * @param column The column to check for options or boolean type.
* @param fetchedColumn The fetched column to check for the type in the external database. * @param fetchedColumn The fetched column to check for the type in the external database.
*/ */
export function shouldCopySpecialColumn( function shouldCopySpecialColumn(
column: { type: string }, column: { type: FieldType },
fetchedColumn: { type: string } | undefined fetchedColumn: { type: FieldType } | undefined
) { ) {
const isFormula = column.type === FieldType.FORMULA const isFormula = column.type === FieldType.FORMULA
const specialTypes = [
FieldType.OPTIONS,
FieldType.LONGFORM,
FieldType.ARRAY,
FieldType.FORMULA,
FieldType.BB_REFERENCE,
]
// column has been deleted, remove - formulas will never exist, always copy // column has been deleted, remove - formulas will never exist, always copy
if (!isFormula && column && !fetchedColumn) { if (!isFormula && column && !fetchedColumn) {
return false return false
} }
const fetchedIsNumber = const fetchedIsNumber =
!fetchedColumn || fetchedColumn.type === FieldType.NUMBER !fetchedColumn || fetchedColumn.type === FieldType.NUMBER
return ( return fetchedIsNumber && column.type === FieldType.BOOLEAN
specialTypes.indexOf(column.type as FieldType) !== -1 ||
(fetchedIsNumber && column.type === FieldType.BOOLEAN)
)
} }
/** /**
@ -357,11 +347,44 @@ function copyExistingPropsOver(
continue continue
} }
const column = existingTableSchema[key] const column = existingTableSchema[key]
const existingColumnType = column?.type
const updatedColumnType = table.schema[key]?.type
// If the db column type changed to a non-compatible one, we want to re-fetch it
if ( if (
shouldCopyRelationship(column, tableIds) || updatedColumnType !== existingColumnType &&
shouldCopySpecialColumn(column, table.schema[key]) !SWITCHABLE_TYPES[updatedColumnType]?.includes(existingColumnType)
) { ) {
table.schema[key] = existingTableSchema[key] continue
}
if (
column.type === FieldType.LINK &&
!shouldCopyRelationship(column, tableIds)
) {
continue
}
const specialTypes = [
FieldType.OPTIONS,
FieldType.LONGFORM,
FieldType.ARRAY,
FieldType.FORMULA,
FieldType.BB_REFERENCE,
]
if (
specialTypes.includes(column.type) &&
!shouldCopySpecialColumn(column, table.schema[key])
) {
continue
}
table.schema[key] = {
...existingTableSchema[key],
externalType:
existingTableSchema[key].externalType ||
table.schema[key].externalType,
} }
} }
} }

View File

@ -348,8 +348,7 @@ const preSaveAction: Partial<Record<SourceName, any>> = {
* Make sure all datasource entities have a display name selected * Make sure all datasource entities have a display name selected
*/ */
export function setDefaultDisplayColumns(datasource: Datasource) { export function setDefaultDisplayColumns(datasource: Datasource) {
// for (const entity of Object.values(datasource.entities || {})) {
for (let entity of Object.values(datasource.entities || {})) {
if (entity.primaryDisplay) { if (entity.primaryDisplay) {
continue continue
} }

View File

@ -3,7 +3,6 @@ import {
Operation, Operation,
RelationshipType, RelationshipType,
RenameColumn, RenameColumn,
AddColumn,
Table, Table,
TableRequest, TableRequest,
ViewV2, ViewV2,
@ -33,7 +32,7 @@ import * as viewSdk from "../../views"
export async function save( export async function save(
datasourceId: string, datasourceId: string,
update: Table, update: Table,
opts?: { tableId?: string; renaming?: RenameColumn; adding?: AddColumn } opts?: { tableId?: string; renaming?: RenameColumn }
) { ) {
let tableToSave: TableRequest = { let tableToSave: TableRequest = {
...update, ...update,
@ -179,14 +178,7 @@ export async function save(
// remove the rename prop // remove the rename prop
delete tableToSave._rename delete tableToSave._rename
// if adding a new column, we need to rebuild the schema for that table to get the 'externalType' of the column datasource.entities[tableToSave.name] = tableToSave
if (opts?.adding) {
datasource.entities[tableToSave.name] = (
await datasourceSdk.buildFilteredSchema(datasource, [tableToSave.name])
).tables[tableToSave.name]
} else {
datasource.entities[tableToSave.name] = tableToSave
}
// store it into couch now for budibase reference // store it into couch now for budibase reference
await db.put(populateExternalTableSchemas(datasource)) await db.put(populateExternalTableSchemas(datasource))

View File

@ -0,0 +1,33 @@
import { FieldType } from "@budibase/types"
type SwitchableTypes = Partial<{
[K in FieldType]: [K, ...FieldType[]]
}>
export const SWITCHABLE_TYPES: SwitchableTypes = {
[FieldType.STRING]: [
FieldType.STRING,
FieldType.OPTIONS,
FieldType.LONGFORM,
FieldType.BARCODEQR,
],
[FieldType.OPTIONS]: [
FieldType.OPTIONS,
FieldType.STRING,
FieldType.LONGFORM,
FieldType.BARCODEQR,
],
[FieldType.LONGFORM]: [
FieldType.LONGFORM,
FieldType.STRING,
FieldType.OPTIONS,
FieldType.BARCODEQR,
],
[FieldType.BARCODEQR]: [
FieldType.BARCODEQR,
FieldType.STRING,
FieldType.OPTIONS,
FieldType.LONGFORM,
],
[FieldType.NUMBER]: [FieldType.NUMBER, FieldType.BOOLEAN],
}

View File

@ -1,4 +1,5 @@
export * from "./api" export * from "./api"
export * from "./fields"
export const OperatorOptions = { export const OperatorOptions = {
Equals: { Equals: {

View File

@ -1,6 +1,6 @@
import { Document } from "../../document" import { Document } from "../../document"
import { View, ViewV2 } from "../view" import { View, ViewV2 } from "../view"
import { AddColumn, RenameColumn } from "../../../sdk" import { RenameColumn } from "../../../sdk"
import { TableSchema } from "./schema" import { TableSchema } from "./schema"
export const INTERNAL_TABLE_SOURCE_ID = "bb_internal" export const INTERNAL_TABLE_SOURCE_ID = "bb_internal"
@ -30,6 +30,5 @@ export interface Table extends Document {
export interface TableRequest extends Table { export interface TableRequest extends Table {
_rename?: RenameColumn _rename?: RenameColumn
_add?: AddColumn
created?: boolean created?: boolean
} }

View File

@ -77,10 +77,6 @@ export interface RenameColumn {
updated: string updated: string
} }
export interface AddColumn {
name: string
}
export interface RelationshipsJson { export interface RelationshipsJson {
through?: string through?: string
from?: string from?: string