MirrorSave
/
budibase
mirror of https://github.com/Budibase/budibase.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Merge branch 'master' into fix/default-table-sqs-indexing

This commit is contained in:
Michael Drury 2024-07-05 16:32:14 +01:00 committed by GitHub
parent eafe66d01e 3ba3d3485a
commit e8ceed08a5
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
13 changed files with 192 additions and 37 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

5
packages/backend-core/src/db/constants.ts
View File

@ -1,5 +0,0 @@
export {
CONSTANT_INTERNAL_ROW_COLS,
CONSTANT_EXTERNAL_ROW_COLS,
isInternalColumnName,
} from "@budibase/shared-core"

3
packages/backend-core/src/db/couch/DatabaseImpl.ts
View File

@ -13,6 +13,7 @@ import {
isDocument,
RowResponse,
RowValue,
SqlClient,
SQLiteDefinition,
SqlQueryBinding,
} from "@budibase/types"
@ -25,6 +26,7 @@ import { SQLITE_DESIGN_DOC_ID } from "../../constants"
import { DDInstrumentedDatabase } from "../instrumentation"
import { checkSlashesInUrl } from "../../helpers"
import env from "../../environment"
import { sqlLog } from "../../sql/utils"
const DATABASE_NOT_FOUND = "Database does not exist."
@ -322,6 +324,7 @@ export class DatabaseImpl implements Database {
): Promise<T[]> {
const dbName = this.name
const url = `/${dbName}/${SQLITE_DESIGN_DOC_ID}`
sqlLog(SqlClient.SQL_LITE, sql, parameters)
return await this._sqlQuery<T[]>(url, "POST", {
query: sql,
args: parameters,

1
packages/backend-core/src/db/couch/index.ts
View File

@ -2,4 +2,3 @@ export * from "./connections"
export * from "./DatabaseImpl"
export * from "./utils"
export { init, getPouch, getPouchDB, closePouchDB } from "./pouchDB"
export * from "../constants"

22
packages/backend-core/src/sql/sql.ts
View File

@ -3,8 +3,10 @@ import * as dbCore from "../db"
import {
getNativeSql,
isExternalTable,
isIsoDateString,
isValidISODateString,
isValidFilter,
sqlLog,
isInvalidISODateString,
} from "./utils"
import { SqlStatements } from "./sqlStatements"
import SqlTableQueryBuilder from "./sqlTable"
@ -38,10 +40,6 @@ const envLimit = environment.SQL_MAX_ROWS
: null
const BASE_LIMIT = envLimit || 5000
// these are invalid dates sent by the client, need to convert them to a real max date
const MIN_ISO_DATE = "0000-00-00T00:00:00.000Z"
const MAX_ISO_DATE = "9999-00-00T00:00:00.000Z"
function likeKey(client: string, key: string): string {
let start: string, end: string
switch (client) {
@ -75,10 +73,10 @@ function parse(input: any) {
if (typeof input !== "string") {
return input
}
if (input === MAX_ISO_DATE || input === MIN_ISO_DATE) {
if (isInvalidISODateString(input)) {
return null
}
if (isIsoDateString(input)) {
if (isValidISODateString(input)) {
return new Date(input.trim())
}
return input
@ -938,15 +936,7 @@ class SqlQueryBuilder extends SqlTableQueryBuilder {
}
log(query: string, values?: SqlQueryBinding) {
if (!environment.SQL_LOGGING_ENABLE) {
return
}
const sqlClient = this.getSqlClient()
let string = `[SQL] [${sqlClient.toUpperCase()}] query="${query}"`
if (values) {
string += ` values="${values.join(", ")}"`
}
console.log(string)
sqlLog(this.getSqlClient(), query, values)
}
}

29
packages/backend-core/src/sql/utils.ts
View File

@ -2,10 +2,12 @@ import { DocumentType, SqlQuery, Table, TableSourceType } from "@budibase/types"
import { DEFAULT_BB_DATASOURCE_ID } from "../constants"
import { Knex } from "knex"
import { SEPARATOR } from "../db"
import environment from "../environment"
const DOUBLE_SEPARATOR = `${SEPARATOR}${SEPARATOR}`
const ROW_ID_REGEX = /^\[.*]$/g
const ENCODED_SPACE = encodeURIComponent(" ")
const ISO_DATE_REGEX = /^\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}.\d{3}Z$/
export function isExternalTableID(tableId: string) {
return tableId.startsWith(DocumentType.DATASOURCE + SEPARATOR)
@ -120,15 +122,38 @@ export function breakRowIdField(_id: string | { _id: string }): any[] {
}
}
export function isIsoDateString(str: string) {
export function isInvalidISODateString(str: string) {
const trimmedValue = str.trim()
if (!/^\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}.\d{3}Z$/.test(trimmedValue)) {
if (!ISO_DATE_REGEX.test(trimmedValue)) {
return false
}
let d = new Date(trimmedValue)
return isNaN(d.getTime())
}
export function isValidISODateString(str: string) {
const trimmedValue = str.trim()
if (!ISO_DATE_REGEX.test(trimmedValue)) {
return false
}
let d = new Date(trimmedValue)
if (isNaN(d.getTime())) {
return false
}
return d.toISOString() === trimmedValue
}
export function isValidFilter(value: any) {
return value != null && value !== ""
}
export function sqlLog(client: string, query: string, values?: any[]) {
if (!environment.SQL_LOGGING_ENABLE) {
return
}
let string = `[SQL] [${client.toUpperCase()}] query="${query}"`
if (values) {
string += ` values="${values.join(", ")}"`
}
console.log(string)
}

9
packages/backend-core/tests/core/utilities/jestUtils.ts
View File

@ -1,4 +1,7 @@
import { db } from "../../../src"
import {
CONSTANT_EXTERNAL_ROW_COLS,
CONSTANT_INTERNAL_ROW_COLS,
} from "@budibase/shared-core"
export function expectFunctionWasCalledTimesWith(
jestFunction: any,
@ -11,7 +14,7 @@ export function expectFunctionWasCalledTimesWith(
}
export const expectAnyInternalColsAttributes: {
[K in (typeof db.CONSTANT_INTERNAL_ROW_COLS)[number]]: any
[K in (typeof CONSTANT_INTERNAL_ROW_COLS)[number]]: any
} = {
tableId: expect.anything(),
type: expect.anything(),
@ -22,7 +25,7 @@ export const expectAnyInternalColsAttributes: {
}
export const expectAnyExternalColsAttributes: {
[K in (typeof db.CONSTANT_EXTERNAL_ROW_COLS)[number]]: any
[K in (typeof CONSTANT_EXTERNAL_ROW_COLS)[number]]: any
} = {
tableId: expect.anything(),
_id: expect.anything(),

4
packages/builder/src/components/backend/DataTable/TableDataTable.svelte
View File

@ -5,6 +5,7 @@
import { TableNames } from "constants"
import { Grid } from "@budibase/frontend-core"
import { API } from "api"
import GridCreateAutomationButton from "./buttons/grid/GridCreateAutomationButton.svelte"
import GridAddColumnModal from "components/backend/DataTable/modals/grid/GridCreateColumnModal.svelte"
import GridCreateEditRowModal from "components/backend/DataTable/modals/grid/GridCreateEditRowModal.svelte"
import GridEditUserModal from "components/backend/DataTable/modals/grid/GridEditUserModal.svelte"
@ -81,6 +82,9 @@
<GridCreateViewButton />
{/if}
<GridManageAccessButton />
{#if !isUsersTable}
<GridCreateAutomationButton />
{/if}
{#if relationshipsEnabled}
<GridRelationshipButton />
{/if}

101
packages/builder/src/components/backend/DataTable/buttons/grid/GridCreateAutomationButton.svelte Normal file
View File

@ -0,0 +1,101 @@
<script>
import {
ActionButton,
Popover,
Menu,
MenuItem,
notifications,
} from "@budibase/bbui"
import { getContext } from "svelte"
import { automationStore, tables, builderStore } from "stores/builder"
import { TriggerStepID } from "constants/backend/automations"
import { goto } from "@roxi/routify"
const { datasource } = getContext("grid")
$: triggers = $automationStore.blockDefinitions.TRIGGER
$: table = $tables.list.find(table => table._id === $datasource.tableId)
async function createAutomation(type) {
const triggerType = triggers[type]
if (!triggerType) {
console.error("Invalid trigger type", type)
notifications.error("Invalid automation trigger type")
return
}
if (!table) {
notifications.error("Invalid table, cannot create automation")
return
}
const automationName = `${table.name} : Row ${
type === TriggerStepID.ROW_SAVED ? "created" : "updated"
}`
const triggerBlock = automationStore.actions.constructBlock(
"TRIGGER",
triggerType.stepId,
triggerType
)
triggerBlock.inputs = { tableId: $datasource.tableId }
try {
const response = await automationStore.actions.create(
automationName,
triggerBlock
)
builderStore.setPreviousTopNavPath(
"/builder/app/:application/data",
window.location.pathname
)
$goto(`/builder/app/${response.appId}/automation/${response.id}`)
notifications.success(`Automation created`)
} catch (e) {
console.error("Error creating automation", e)
notifications.error("Error creating automation")
}
}
let anchor
let open
</script>
<div bind:this={anchor}>
<ActionButton
icon="MagicWand"
quiet
size="M"
on:click={() => (open = !open)}
selected={open}
>
Generate
</ActionButton>
</div>
<Popover bind:open {anchor} align="left">
<Menu>
<MenuItem
icon="ShareAndroid"
on:click={() => {
open = false
createAutomation(TriggerStepID.ROW_SAVED)
}}
>
Automation: when row is created
</MenuItem>
<MenuItem
icon="ShareAndroid"
on:click={() => {
open = false
createAutomation(TriggerStepID.ROW_UPDATED)
}}
>
Automation: when row is updated
</MenuItem>
</Menu>
</Popover>
<style>
</style>

29
packages/server/src/api/routes/tests/search.spec.ts
View File

@ -2186,6 +2186,35 @@ describe.each([
})
})
describe.each([
{ low: "2024-07-03T00:00:00.000Z", high: "9999-00-00T00:00:00.000Z" },
{ low: "2024-07-03T00:00:00.000Z", high: "9998-00-00T00:00:00.000Z" },
{ low: "0000-00-00T00:00:00.000Z", high: "2024-07-04T00:00:00.000Z" },
{ low: "0001-00-00T00:00:00.000Z", high: "2024-07-04T00:00:00.000Z" },
])("date special cases", ({ low, high }) => {
const earlyDate = "2024-07-03T10:00:00.000Z",
laterDate = "2024-07-03T11:00:00.000Z"
beforeAll(async () => {
table = await createTable({
date: {
name: "date",
type: FieldType.DATETIME,
},
})
await createRows([{ date: earlyDate }, { date: laterDate }])
})
it("should be able to handle a date search", async () => {
await expectSearch({
query: {
range: {
"1:date": { low, high },
},
},
}).toContainExactly([{ date: earlyDate }, { date: laterDate }])
})
})
describe.each([
"名前", // Japanese for "name"
"Benutzer-ID", // German for "user ID", includes a hyphen

6
packages/server/src/sdk/app/rows/search/external.ts
View File

@ -16,9 +16,9 @@ import {
breakExternalTableId,
breakRowIdField,
} from "../../../../integrations/utils"
import { utils } from "@budibase/shared-core"
import { utils, CONSTANT_EXTERNAL_ROW_COLS } from "@budibase/shared-core"
import { ExportRowsParams, ExportRowsResult } from "./types"
import { db, HTTPError } from "@budibase/backend-core"
import { HTTPError } from "@budibase/backend-core"
import pick from "lodash/pick"
import { outputProcessing } from "../../../../utilities/rowProcessor"
import sdk from "../../../"
@ -99,7 +99,7 @@ export async function search(
}
if (options.fields) {
const fields = [...options.fields, ...db.CONSTANT_EXTERNAL_ROW_COLS]
const fields = [...options.fields, ...CONSTANT_EXTERNAL_ROW_COLS]
rows = rows.map((r: any) => pick(r, fields))
}

5
packages/server/src/sdk/app/rows/search/internal.ts
View File

@ -1,4 +1,5 @@
import { context, db, HTTPError } from "@budibase/backend-core"
import { context, HTTPError } from "@budibase/backend-core"
import { CONSTANT_INTERNAL_ROW_COLS } from "@budibase/shared-core"
import env from "../../../../environment"
import { fullSearch, paginatedSearch } from "./utils"
import { getRowParams, InternalTables } from "../../../../db/utils"
@ -74,7 +75,7 @@ export async function search(
}
if (options.fields) {
const fields = [...options.fields, ...db.CONSTANT_INTERNAL_ROW_COLS]
const fields = [...options.fields, ...CONSTANT_INTERNAL_ROW_COLS]
response.rows = response.rows.map((r: any) => pick(r, fields))
}

3
packages/server/src/sdk/app/tables/migration.ts
View File

@ -18,6 +18,7 @@ import sdk from "../../../sdk"
import { isExternalTableID } from "../../../integrations/utils"
import { EventType, updateLinks } from "../../../db/linkedRows"
import { cloneDeep } from "lodash"
import { isInternalColumnName } from "@budibase/shared-core"
export interface MigrationResult {
tablesUpdated: Table[]
@ -36,7 +37,7 @@ export async function migrate(
throw new BadRequestError(`Column name cannot be empty`)
}
if (dbCore.isInternalColumnName(newColumnName)) {
if (isInternalColumnName(newColumnName)) {
throw new BadRequestError(`Column name cannot be a reserved column name`)
}

12
packages/server/src/sdk/app/views/index.ts
View File

@ -6,9 +6,13 @@ import {
ViewV2,
ViewV2Enriched,
} from "@budibase/types"
import { HTTPError, db as dbCore } from "@budibase/backend-core"
import { HTTPError } from "@budibase/backend-core"
import { features } from "@budibase/pro"
import { helpers } from "@budibase/shared-core"
import {
helpers,
CONSTANT_EXTERNAL_ROW_COLS,
CONSTANT_INTERNAL_ROW_COLS,
} from "@budibase/shared-core"
import { cloneDeep } from "lodash/fp"
import * as utils from "../../../db/utils"
@ -144,8 +148,8 @@ export function allowedFields(view: View | ViewV2) {
const fieldSchema = view.schema![key]
return fieldSchema.visible && !fieldSchema.readonly
}),
...dbCore.CONSTANT_EXTERNAL_ROW_COLS,
...dbCore.CONSTANT_INTERNAL_ROW_COLS,
...CONSTANT_EXTERNAL_ROW_COLS,
...CONSTANT_INTERNAL_ROW_COLS,
]
}