MirrorSave
/
budibase
mirror of https://github.com/Budibase/budibase.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Merge branch 'master' into BUDI-8885/only-select-required-columns-from-sql-databases

This commit is contained in:
Adria Navarro 2024-12-18 12:37:14 +01:00 committed by GitHub
parent b05b523a6e e2aed7e8c4
commit 7d8cfebc34
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
17 changed files with 263 additions and 162 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
packages/builder/package.json
View File

@ -7,7 +7,7 @@
"build": "routify -b && NODE_OPTIONS=\"--max_old_space_size=4096\" vite build --emptyOutDir",
"start": "routify -c rollup",
"dev": "routify -c dev:vite",
"dev:vite": "vite --host 0.0.0.0",
"dev:vite": "vite --host 0.0.0.0 --mode=dev",
"rollup": "rollup -c -w",
"test": "vitest run",
"test:watch": "vitest",

79
packages/builder/src/pages/builder/app/[application]/automation/_layout.svelte
View File

@ -1,5 +1,5 @@
<script>
import { Heading, Body, Layout, Button, Modal, Icon } from "@budibase/bbui"
import { Heading, Body, Layout, Button, Modal } from "@budibase/bbui"
import AutomationPanel from "components/automation/AutomationPanel/AutomationPanel.svelte"
import CreateAutomationModal from "components/automation/AutomationPanel/CreateAutomationModal.svelte"
import CreateWebhookModal from "components/automation/Shared/CreateWebhookModal.svelte"
@ -12,13 +12,10 @@
automationStore,
selectedAutomation,
} from "stores/builder"
import { createLocalStorageStore } from "@budibase/frontend-core"
import { fly } from "svelte/transition"
$: automationId = $selectedAutomation?.data?._id
$: builderStore.selectResource(automationId)
const surveyDismissed = createLocalStorageStore("automation-survey", false)
const stopSyncing = syncURLToState({
urlParam: "automationId",
stateKey: "selectedAutomationId",
@ -31,11 +28,9 @@
let modal
let webhookModal
let mounted = false
onMount(() => {
$automationStore.showTestPanel = false
mounted = true
})
onDestroy(stopSyncing)
@ -83,43 +78,6 @@
</Modal>
</div>
{#if !$surveyDismissed && mounted}
<div
class="survey"
in:fly={{ x: 600, duration: 260, delay: 1000 }}
out:fly={{ x: 600, duration: 260 }}
>
<div class="survey__body">
<div class="survey__title">We value your feedback!</div>
<div class="survey__text">
<a
href="https://t.maze.co/310149185"
target="_blank"
rel="noopener noreferrer"
on:click={() => surveyDismissed.set(true)}
>
Complete our survey on Automations</a
>
and receive a $20 thank-you gift.
<a
href="https://drive.google.com/file/d/12-qk_2F9g5PdbM6wuKoz2KkIyLI-feMX/view?usp=sharing"
target="_blank"
rel="noopener noreferrer"
>
Terms apply.
</a>
</div>
</div>
<Icon
name="Close"
hoverable
color="var(--spectrum-global-color-static-gray-300)"
hoverColor="var(--spectrum-global-color-static-gray-100)"
on:click={() => surveyDismissed.set(true)}
/>
</div>
{/if}
<style>
.root {
flex: 1 1 auto;
@ -164,39 +122,4 @@
grid-column: 3;
overflow: auto;
}
/* Survey */
.survey {
position: absolute;
bottom: 32px;
right: 32px;
background: var(--spectrum-semantic-positive-color-background);
display: flex;
flex-direction: row;
padding: var(--spacing-l) var(--spacing-xl);
border-radius: 4px;
gap: var(--spacing-xl);
}
.survey * {
color: var(--spectrum-global-color-static-gray-300);
white-space: nowrap;
}
.survey a {
text-decoration: underline;
transition: color 130ms ease-out;
}
.survey a:hover {
color: var(--spectrum-global-color-static-gray-100);
cursor: pointer;
}
.survey__body {
flex: 1 1 auto;
display: flex;
flex-direction: column;
gap: 2px;
}
.survey__title {
font-weight: 600;
font-size: 15px;
}
</style>

112
packages/builder/src/stores/builder/datasources.js → packages/builder/src/stores/builder/datasources.ts
View File

@ -7,11 +7,26 @@ import {
import { tables } from "./tables"
import { queries } from "./queries"
import { API } from "api"
import { DatasourceFeature } from "@budibase/types"
import {
DatasourceFeature,
Datasource,
Table,
Integration,
UIIntegration,
SourceName,
} from "@budibase/types"
// @ts-ignore
import { TableNames } from "constants"
// when building the internal DS - seems to represent it slightly differently to the backend typing of a DS
interface InternalDatasource extends Omit<Datasource, "entities"> {
entities: Table[]
}
class TableImportError extends Error {
constructor(errors) {
errors: Record<string, string>
constructor(errors: Record<string, string>) {
super()
this.name = "TableImportError"
this.errors = errors
@ -26,8 +41,13 @@ class TableImportError extends Error {
}
}
interface DatasourceStore {
list: Datasource[]
selectedDatasourceId: null | string
}
export function createDatasourcesStore() {
const store = writable({
const store = writable<DatasourceStore>({
list: [],
selectedDatasourceId: null,
})
@ -36,23 +56,25 @@ export function createDatasourcesStore() {
// Set the internal datasource entities from the table list, which we're
// able to keep updated unlike the egress generated definition of the
// internal datasource
let internalDS = $store.list?.find(ds => ds._id === BUDIBASE_INTERNAL_DB_ID)
let internalDS: Datasource | InternalDatasource | undefined =
$store.list?.find(ds => ds._id === BUDIBASE_INTERNAL_DB_ID)
let otherDS = $store.list?.filter(ds => ds._id !== BUDIBASE_INTERNAL_DB_ID)
if (internalDS) {
const tables: Table[] = $tables.list?.filter((table: Table) => {
return (
table.sourceId === BUDIBASE_INTERNAL_DB_ID &&
table._id !== TableNames.USERS
)
})
internalDS = {
...internalDS,
entities: $tables.list?.filter(table => {
return (
table.sourceId === BUDIBASE_INTERNAL_DB_ID &&
table._id !== TableNames.USERS
)
}),
entities: tables,
}
}
// Build up enriched DS list
// Only add the internal DS if we have at least one non-users table
let list = []
let list: (InternalDatasource | Datasource)[] = []
if (internalDS?.entities?.length) {
list.push(internalDS)
}
@ -75,62 +97,82 @@ export function createDatasourcesStore() {
}))
}
const select = id => {
const select = (id: string) => {
store.update(state => ({
...state,
selectedDatasourceId: id,
}))
}
const updateDatasource = (response, { ignoreErrors } = {}) => {
const updateDatasource = (
response: { datasource: Datasource; errors?: Record<string, string> },
{ ignoreErrors }: { ignoreErrors?: boolean } = {}
) => {
const { datasource, errors } = response
if (!ignoreErrors && errors && Object.keys(errors).length > 0) {
throw new TableImportError(errors)
}
replaceDatasource(datasource._id, datasource)
select(datasource._id)
replaceDatasource(datasource._id!, datasource)
select(datasource._id!)
return datasource
}
const updateSchema = async (datasource, tablesFilter) => {
const updateSchema = async (
datasource: Datasource,
tablesFilter: string[]
) => {
const response = await API.buildDatasourceSchema(
datasource?._id,
datasource?._id!,
tablesFilter
)
updateDatasource(response)
}
const sourceCount = source => {
const sourceCount = (source: string) => {
return get(store).list.filter(datasource => datasource.source === source)
.length
}
const checkDatasourceValidity = async (integration, datasource) => {
const checkDatasourceValidity = async (
integration: Integration,
datasource: Datasource
): Promise<{ valid: boolean; error?: string }> => {
if (integration.features?.[DatasourceFeature.CONNECTION_CHECKING]) {
const { connected, error } = await API.validateDatasource(datasource)
if (connected) {
return
return { valid: true }
} else {
return { valid: false, error }
}
throw new Error(`Unable to connect: ${error}`)
}
return { valid: true }
}
const create = async ({ integration, config }) => {
const create = async ({
integration,
config,
}: {
integration: UIIntegration
config: Record<string, any>
}) => {
const count = sourceCount(integration.name)
const nameModifier = count === 0 ? "" : ` ${count + 1}`
const datasource = {
const datasource: Datasource = {
type: "datasource",
source: integration.name,
source: integration.name as SourceName,
config,
name: `${integration.friendlyName}${nameModifier}`,
plus: integration.plus && integration.name !== IntegrationTypes.REST,
isSQL: integration.isSQL,
}
if (await checkDatasourceValidity(integration, datasource)) {
throw new Error("Unable to connect")
const { valid, error } = await checkDatasourceValidity(
integration,
datasource
)
if (!valid) {
throw new Error(`Unable to connect - ${error}`)
}
const response = await API.createDatasource({
@ -141,7 +183,13 @@ export function createDatasourcesStore() {
return updateDatasource(response, { ignoreErrors: true })
}
const update = async ({ integration, datasource }) => {
const update = async ({
integration,
datasource,
}: {
integration: Integration
datasource: Datasource
}) => {
if (await checkDatasourceValidity(integration, datasource)) {
throw new Error("Unable to connect")
}
@ -151,15 +199,15 @@ export function createDatasourcesStore() {
return updateDatasource(response)
}
const deleteDatasource = async datasource => {
const deleteDatasource = async (datasource: Datasource) => {
if (!datasource?._id || !datasource?._rev) {
return
}
await API.deleteDatasource(datasource._id, datasource._rev)
replaceDatasource(datasource._id, null)
replaceDatasource(datasource._id)
}
const replaceDatasource = (datasourceId, datasource) => {
const replaceDatasource = (datasourceId: string, datasource?: Datasource) => {
if (!datasourceId) {
return
}
@ -197,7 +245,7 @@ export function createDatasourcesStore() {
}
}
const getTableNames = async datasource => {
const getTableNames = async (datasource: Datasource) => {
const info = await API.fetchInfoForDatasource(datasource)
return info.tableNames || []
}

39
packages/builder/src/stores/builder/sortedIntegrations.js
View File

@ -1,39 +0,0 @@
import { integrations } from "./integrations"
import { derived } from "svelte/store"
import { DatasourceTypes } from "constants/backend"
const getIntegrationOrder = type => {
if (type === DatasourceTypes.API) return 1
if (type === DatasourceTypes.RELATIONAL) return 2
if (type === DatasourceTypes.NON_RELATIONAL) return 3
// Sort all others arbitrarily by the first character of their name.
// Character codes can technically be as low as 0, so make sure the number is at least 4
return type.charCodeAt(0) + 4
}
export const createSortedIntegrationsStore = () => {
return derived(integrations, $integrations => {
const integrationsAsArray = Object.entries($integrations).map(
([name, integration]) => ({
name,
...integration,
})
)
return integrationsAsArray.sort((integrationA, integrationB) => {
const integrationASortOrder = getIntegrationOrder(integrationA.type)
const integrationBSortOrder = getIntegrationOrder(integrationB.type)
if (integrationASortOrder === integrationBSortOrder) {
return integrationA.friendlyName.localeCompare(
integrationB.friendlyName
)
}
return integrationASortOrder < integrationBSortOrder ? -1 : 1
})
})
}
export const sortedIntegrations = createSortedIntegrationsStore()

46
packages/builder/src/stores/builder/sortedIntegrations.ts Normal file
View File

@ -0,0 +1,46 @@
import { integrations } from "./integrations"
import { derived } from "svelte/store"
import { DatasourceTypes } from "constants/backend"
import { UIIntegration, Integration } from "@budibase/types"
const getIntegrationOrder = (type: string | undefined) => {
// if type is not known, sort to end
if (!type) {
return Number.MAX_SAFE_INTEGER
}
if (type === DatasourceTypes.API) return 1
if (type === DatasourceTypes.RELATIONAL) return 2
if (type === DatasourceTypes.NON_RELATIONAL) return 3
// Sort all others arbitrarily by the first character of their name.
// Character codes can technically be as low as 0, so make sure the number is at least 4
return type.charCodeAt(0) + 4
}
export const createSortedIntegrationsStore = () => {
return derived<typeof integrations, UIIntegration[]>(
integrations,
$integrations => {
const entries: [string, Integration][] = Object.entries($integrations)
const integrationsAsArray = entries.map(([name, integration]) => ({
name,
...integration,
}))
return integrationsAsArray.sort((integrationA, integrationB) => {
const integrationASortOrder = getIntegrationOrder(integrationA.type)
const integrationBSortOrder = getIntegrationOrder(integrationB.type)
if (integrationASortOrder === integrationBSortOrder) {
return integrationA.friendlyName.localeCompare(
integrationB.friendlyName
)
}
return integrationASortOrder < integrationBSortOrder ? -1 : 1
})
}
)
}
export const sortedIntegrations = createSortedIntegrationsStore()

3
packages/frontend-core/src/components/grid/cells/RelationshipCell.svelte
View File

@ -102,9 +102,8 @@
lastSearchId = Math.random()
searching = true
const thisSearchId = lastSearchId
const results = await searchFunction({
const results = await searchFunction(schema.tableId, {
paginate: false,
tableId: schema.tableId,
limit: 20,
query: {
string: {

4
packages/server/scripts/integrations/mysql/init.sql
View File

@ -30,6 +30,10 @@ CREATE TABLE Products (
name text,
updated time
);
CREATE TABLE `table with space` (
id serial primary key,
name text
);
INSERT INTO Persons (FirstName, LastName, Age, Address, City, CreatedAt) VALUES ('Mike', 'Hughes', 28.2, '123 Fake Street', 'Belfast', '2021-01-19 03:14:07');
INSERT INTO Persons (FirstName, LastName, Age, Address, City, CreatedAt) VALUES ('Dave', 'Johnson', 29, '124 Fake Street', 'Belfast', '2022-04-01 00:11:11');
INSERT INTO Person (Name) VALUES ('Elf');

20
packages/server/src/api/controllers/row/external.ts
View File

@ -55,10 +55,22 @@ export async function patch(ctx: UserCtx<PatchRowRequest, PatchRowResponse>) {
const table = await utils.getTableFromSource(source)
const { _id, ...rowData } = ctx.request.body
const dataToUpdate = await inputProcessing(
const beforeRow = await sdk.rows.external.getRow(table._id!, _id, {
relationships: true,
})
let dataToUpdate = cloneDeep(beforeRow)
const allowedField = utils.getSourceFields(source)
for (const key of Object.keys(rowData)) {
if (!allowedField.includes(key)) continue
dataToUpdate[key] = rowData[key]
}
dataToUpdate = await inputProcessing(
ctx.user?._id,
cloneDeep(source),
rowData
dataToUpdate
)
const validateResult = await sdk.rows.utils.validate({
@ -69,10 +81,6 @@ export async function patch(ctx: UserCtx<PatchRowRequest, PatchRowResponse>) {
throw { validation: validateResult.errors }
}
const beforeRow = await sdk.rows.external.getRow(table._id!, _id, {
relationships: true,
})
const response = await handleRequest(Operation.UPDATE, source, {
id: breakRowIdField(_id),
row: dataToUpdate,

15
packages/server/src/api/controllers/row/utils/utils.ts
View File

@ -110,6 +110,21 @@ function fixBooleanFields(row: Row, table: Table) {
return row
}
export function getSourceFields(source: Table | ViewV2): string[] {
const isView = sdk.views.isView(source)
if (isView) {
const fields = Object.keys(
helpers.views.basicFields(source, { visible: true })
)
return fields
}
const fields = Object.entries(source.schema)
.filter(([_, field]) => field.visible !== false)
.map(([columnName]) => columnName)
return fields
}
export async function sqlOutputProcessing(
rows: DatasourcePlusQueryResponse,
source: Table | ViewV2,

56
packages/server/src/api/routes/tests/row.spec.ts
View File

@ -1333,6 +1333,62 @@ if (descriptions.length) {
expect(resp.relationship.length).toBe(1)
})
it("should be able to keep linked data when updating from views that trims links from the main table", async () => {
let row = await config.api.row.save(table._id!, {
name: "main",
description: "main description",
})
const row2 = await config.api.row.save(otherTable._id!, {
name: "link",
description: "link description",
relationship: [row._id],
})
const view = await config.api.viewV2.create({
tableId: table._id!,
name: "view",
schema: {
name: { visible: true },
},
})
const resp = await config.api.row.patch(view.id, {
_id: row._id!,
_rev: row._rev!,
tableId: row.tableId!,
name: "test2",
relationship: [row2._id],
})
expect(resp.relationship).toBeUndefined()
const updatedRow = await config.api.row.get(table._id!, row._id!)
expect(updatedRow.relationship.length).toBe(1)
})
it("should be able to keep linked data when updating from views that trims links from the foreign table", async () => {
let row = await config.api.row.save(table._id!, {
name: "main",
description: "main description",
})
const row2 = await config.api.row.save(otherTable._id!, {
name: "link",
description: "link description",
relationship: [row._id],
})
const view = await config.api.viewV2.create({
tableId: otherTable._id!,
name: "view",
})
await config.api.row.patch(view.id, {
_id: row2._id!,
_rev: row2._rev!,
tableId: row2.tableId!,
})
const updatedRow = await config.api.row.get(table._id!, row._id!)
expect(updatedRow.relationship.length).toBe(1)
})
!isInternal &&
// MSSQL needs a setting called IDENTITY_INSERT to be set to ON to allow writing
// to identity columns. This is not something Budibase does currently.

32
packages/server/src/api/routes/tests/viewV2.spec.ts
View File

@ -55,7 +55,7 @@ if (descriptions.length) {
let datasource: Datasource | undefined
function saveTableRequest(
...overrides: Partial<Omit<SaveTableRequest, "name">>[]
...overrides: Partial<SaveTableRequest>[]
): SaveTableRequest {
const req: SaveTableRequest = {
name: generator.guid().replaceAll("-", "").substring(0, 16),
@ -1898,6 +1898,36 @@ if (descriptions.length) {
}
expect(view.queryUI).toEqual(expected)
})
it("tables and views can contain whitespaces", async () => {
const table = await config.api.table.save(
saveTableRequest({
name: `table with spaces ${generator.hash()}`,
schema: {
name: {
type: FieldType.STRING,
name: "name",
},
},
})
)
const view = await config.api.viewV2.create({
tableId: table._id!,
name: `view name with spaces`,
schema: {
name: { visible: true },
},
})
expect(await getDelegate(view)).toEqual({
...view,
schema: {
id: { ...table.schema["id"], visible: false },
name: { ...table.schema["name"], visible: true },
},
})
})
})
describe("updating table schema", () => {

7
packages/server/src/tests/utilities/api/viewV2.ts
View File

@ -46,8 +46,11 @@ export class ViewV2API extends TestAPI {
}
get = async (viewId: string) => {
return (await this._get<ViewResponseEnriched>(`/api/v2/views/${viewId}`))
.data
return (
await this._get<ViewResponseEnriched>(
`/api/v2/views/${encodeURIComponent(viewId)}`
)
).data
}
fetch = async (expectations?: Expectations) => {

2
packages/types/src/api/web/app/datasource.ts
View File

@ -12,7 +12,7 @@ export interface UpdateDatasourceResponse {
export interface CreateDatasourceRequest {
datasource: Datasource
fetchSchema?: boolean
tablesFilter: string[]
tablesFilter?: string[]
}
export interface VerifyDatasourceRequest {

1
packages/types/src/index.ts
View File

@ -3,3 +3,4 @@ export * from "./sdk"
export * from "./api"
export * from "./core"
export * from "./shared"
export * from "./ui"

1
packages/types/src/ui/index.ts Normal file
View File

@ -0,0 +1 @@
export * from "./stores"

1
packages/types/src/ui/stores/index.ts Normal file
View File

@ -0,0 +1 @@
export * from "./integration"

5
packages/types/src/ui/stores/integration.ts Normal file
View File

@ -0,0 +1,5 @@
import { Integration } from "@budibase/types"
export interface UIIntegration extends Integration {
name: string
}