MirrorSave
/
budibase
mirror of https://github.com/Budibase/budibase.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Merge branch 'master' into revert-13487-revert-13463-BUDI-8157

This commit is contained in:
Mike Sealey 2024-04-19 16:11:54 +01:00 committed by GitHub
parent 4c0d3ed5f3 39053d7f88
commit 2d3fcae2e9
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
125 changed files with 1850 additions and 1251 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

25
.eslintrc.json
View File

@ -42,7 +42,17 @@
},
"rules": {
"no-unused-vars": "off",
"@typescript-eslint/no-unused-vars": "error",
"local-rules/no-budibase-imports": "error",
"local-rules/no-console-error": "error",
"@typescript-eslint/no-unused-vars": [
"error",
{
"varsIgnorePattern": "^_",
"argsIgnorePattern": "^_",
"destructuredArrayIgnorePattern": "^_",
"ignoreRestSiblings": true
}
],
"local-rules/no-budibase-imports": "error"
}
},
@ -59,7 +69,15 @@
},
"rules": {
"no-unused-vars": "off",
"@typescript-eslint/no-unused-vars": "error",
"@typescript-eslint/no-unused-vars": [
"error",
{
"varsIgnorePattern": "^_",
"argsIgnorePattern": "^_",
"destructuredArrayIgnorePattern": "^_",
"ignoreRestSiblings": true
}
],
"local-rules/no-test-com": "error",
"local-rules/email-domain-example-com": "error",
"no-console": "warn",
@ -89,7 +107,8 @@
{
"varsIgnorePattern": "^_",
"argsIgnorePattern": "^_",
"destructuredArrayIgnorePattern": "^_"
"destructuredArrayIgnorePattern": "^_",
"ignoreRestSiblings": true
}
],
"import/no-relative-packages": "error",

3
.vscode/settings.json vendored
View File

@ -24,5 +24,8 @@
},
"[svelte]": {
"editor.defaultFormatter": "svelte.svelte-vscode"
},
"[handlebars]": {
"editor.formatOnSave": false
}
}

21
eslint-local-rules/index.js
View File

@ -1,4 +1,25 @@
module.exports = {
"no-console-error": {
create: function(context) {
return {
CallExpression(node) {
if (
node.callee.type === "MemberExpression" &&
node.callee.object.name === "console" &&
node.callee.property.name === "error" &&
node.arguments.length === 1 &&
node.arguments[0].name &&
node.arguments[0].name.startsWith("err")
) {
context.report({
node,
message: 'Using console.error(err) on its own is not allowed. Either provide context to the error (console.error(msg, err)) or throw it.',
})
}
},
};
},
},
"no-budibase-imports": {
create: function (context) {
return {

3
hosting/.env
View File

@ -17,6 +17,7 @@ APP_PORT=4002
WORKER_PORT=4003
MINIO_PORT=4004
COUCH_DB_PORT=4005
COUCH_DB_SQS_PORT=4006
REDIS_PORT=6379
WATCHTOWER_PORT=6161
BUDIBASE_ENVIRONMENT=PRODUCTION
@ -28,4 +29,4 @@ BB_ADMIN_USER_PASSWORD=
# A path that is watched for plugin bundles. Any bundles found are imported automatically/
PLUGINS_DIR=
ROLLING_LOG_MAX_SIZE=
ROLLING_LOG_MAX_SIZE=

2
lerna.json
View File

@ -1,5 +1,5 @@
{
"version": "2.23.5",
"version": "2.23.10",
"npmClient": "yarn",
"packages": [
"packages/*",

5
nx.json
View File

@ -9,10 +9,7 @@
},
"targetDefaults": {
"build": {
"inputs": [
"{workspaceRoot}/scripts/build.js",
"{workspaceRoot}/lerna.json"
]
"inputs": ["{workspaceRoot}/scripts/*", "{workspaceRoot}/lerna.json"]
}
}
}

3
package.json
View File

@ -56,9 +56,10 @@
"dev:noserver": "yarn run kill-builder && lerna run --stream dev:stack:up --ignore @budibase/account-portal-server && lerna run --stream dev --ignore @budibase/backend-core --ignore @budibase/server --ignore @budibase/worker --ignore=@budibase/account-portal-ui --ignore @budibase/account-portal-server",
"dev:server": "yarn run kill-server && lerna run --stream dev --scope @budibase/worker --scope @budibase/server",
"dev:accountportal": "yarn kill-accountportal && lerna run dev --stream --scope @budibase/account-portal-ui --scope @budibase/account-portal-server",
"dev:camunda": "./scripts/deploy-camunda.sh",
"dev:all": "yarn run kill-all && lerna run --stream dev",
"dev:built": "yarn run kill-all && cd packages/server && yarn dev:stack:up && cd ../../ && lerna run --stream dev:built",
"dev:docker": "yarn build --scope @budibase/server --scope @budibase/worker && docker-compose -f hosting/docker-compose.build.yaml -f hosting/docker-compose.dev.yaml --env-file hosting/.env up --build --scale proxy-service=0",
"dev:docker": "./scripts/devDocker.sh",
"test": "REUSE_CONTAINERS=1 lerna run --concurrency 1 --stream test --stream",
"lint:eslint": "eslint packages --max-warnings=0",
"lint:prettier": "prettier --check \"packages/**/*.{js,ts,svelte}\" && prettier --write \"examples/**/*.{js,ts,svelte}\"",

2
packages/account-portal

@ -1 +1 @@
Subproject commit bd0e01d639ec3b2547e7c859a1c43b622dce8344
Subproject commit c167c331ff9b8161fc18e2ecbaaf1ea5815ba964

2
packages/backend-core/src/auth/auth.ts
View File

@ -64,7 +64,6 @@ async function refreshOIDCAccessToken(
}
strategy = await oidc.strategyFactory(enrichedConfig, ssoSaveUserNoOp)
} catch (err) {
console.error(err)
throw new Error("Could not refresh OAuth Token")
}
@ -99,7 +98,6 @@ async function refreshGoogleAccessToken(
ssoSaveUserNoOp
)
} catch (err: any) {
console.error(err)
throw new Error(
`Error constructing OIDC refresh strategy: message=${err.message}`
)

14
packages/backend-core/src/db/lucene.ts
View File

@ -8,19 +8,9 @@ import {
SearchParams,
WithRequired,
} from "@budibase/types"
import { dataFilters } from "@budibase/shared-core"
const QUERY_START_REGEX = /\d[0-9]*:/g
export function removeKeyNumbering(key: any): string {
if (typeof key === "string" && key.match(QUERY_START_REGEX) != null) {
const parts = key.split(":")
// remove the number
parts.shift()
return parts.join(":")
} else {
return key
}
}
export const removeKeyNumbering = dataFilters.removeKeyNumbering
/**
* Class to build lucene query URLs.

2
packages/backend-core/src/environment.ts
View File

@ -107,7 +107,7 @@ const environment = {
ENCRYPTION_KEY: process.env.ENCRYPTION_KEY,
API_ENCRYPTION_KEY: getAPIEncryptionKey(),
COUCH_DB_URL: process.env.COUCH_DB_URL || "http://localhost:4005",
COUCH_DB_SQL_URL: process.env.COUCH_DB_SQL_URL || "http://localhost:4984",
COUCH_DB_SQL_URL: process.env.COUCH_DB_SQL_URL || "http://localhost:4006",
COUCH_DB_USERNAME: process.env.COUCH_DB_USER,
COUCH_DB_PASSWORD: process.env.COUCH_DB_PASSWORD,
GOOGLE_CLIENT_ID: process.env.GOOGLE_CLIENT_ID,

2
packages/backend-core/src/middleware/authenticated.ts
View File

@ -138,7 +138,6 @@ export default function (
} catch (err: any) {
authenticated = false
console.error(`Auth Error: ${err.message}`)
console.error(err)
// remove the cookie as the user does not exist anymore
clearCookie(ctx, Cookie.Auth)
}
@ -187,7 +186,6 @@ export default function (
}
} catch (err: any) {
console.error(`Auth Error: ${err.message}`)
console.error(err)
// invalid token, clear the cookie
if (err?.name === "JsonWebTokenError") {
clearCookie(ctx, Cookie.Auth)

2
packages/backend-core/src/middleware/errorHandling.ts
View File

@ -12,7 +12,7 @@ export async function errorHandling(ctx: any, next: any) {
if (status >= 400 && status < 500) {
console.warn(err)
} else {
console.error(err)
console.error("Got 400 response code", err)
}
let error: APIError = {

1
packages/backend-core/src/middleware/passport/sso/google.ts
View File

@ -68,7 +68,6 @@ export async function strategyFactory(
verify
)
} catch (err: any) {
console.error(err)
throw new Error(`Error constructing google authentication strategy: ${err}`)
}
}

2
packages/backend-core/src/middleware/passport/sso/oidc.ts
View File

@ -103,7 +103,6 @@ export async function strategyFactory(
strategy.name = "oidc"
return strategy
} catch (err: any) {
console.error(err)
throw new Error(`Error constructing OIDC authentication strategy - ${err}`)
}
}
@ -142,7 +141,6 @@ export async function fetchStrategyConfig(
callbackURL: callbackUrl,
}
} catch (err) {
console.error(err)
throw new Error(
`Error constructing OIDC authentication configuration - ${err}`
)

1
packages/backend-core/src/migrations/migrations.ts
View File

@ -26,7 +26,6 @@ export const getMigrationsDoc = async (db: any) => {
if (err.status && err.status === 404) {
return { _id: DocumentType.MIGRATIONS }
} else {
console.error(err)
throw err
}
}

4
packages/backend-core/src/queue/inMemoryQueue.ts
View File

@ -115,7 +115,6 @@ class InMemoryQueue implements Partial<Queue> {
* a JSON message as this is required by Bull.
* @param repeat serves no purpose for the import queue.
*/
// eslint-disable-next-line no-unused-vars
async add(data: any, opts?: JobOptions) {
const jobId = opts?.jobId?.toString()
if (jobId && this._queuedJobIds.has(jobId)) {
@ -166,8 +165,7 @@ class InMemoryQueue implements Partial<Queue> {
return []
}
// eslint-disable-next-line @typescript-eslint/no-unused-vars
async removeJobs(pattern: string) {
async removeJobs(_pattern: string) {
// no-op
}

4
packages/backend-core/src/users/db.ts
View File

@ -50,6 +50,8 @@ type CreateAdminUserOpts = {
hashPassword?: boolean
requirePassword?: boolean
skipPasswordValidation?: boolean
firstName?: string
lastName?: string
}
type FeatureFns = { isSSOEnforced: FeatureFn; isAppBuildersEnabled: FeatureFn }
@ -517,6 +519,8 @@ export class UserDB {
global: true,
},
tenantId,
firstName: opts?.firstName,
lastName: opts?.lastName,
}
if (opts?.ssoId) {
user.ssoId = opts.ssoId

12
packages/backend-core/src/users/users.ts
View File

@ -17,8 +17,8 @@ import {
ContextUser,
CouchFindOptions,
DatabaseQueryOpts,
SearchQuery,
SearchQueryOperators,
SearchFilters,
SearchFilterOperator,
SearchUsersRequest,
User,
} from "@budibase/types"
@ -44,11 +44,11 @@ function removeUserPassword(users: User | User[]) {
return users
}
export function isSupportedUserSearch(query: SearchQuery) {
export function isSupportedUserSearch(query: SearchFilters) {
const allowed = [
{ op: SearchQueryOperators.STRING, key: "email" },
{ op: SearchQueryOperators.EQUAL, key: "_id" },
{ op: SearchQueryOperators.ONE_OF, key: "_id" },
{ op: SearchFilterOperator.STRING, key: "email" },
{ op: SearchFilterOperator.EQUAL, key: "_id" },
{ op: SearchFilterOperator.ONE_OF, key: "_id" },
]
for (let [key, operation] of Object.entries(query)) {
if (typeof operation !== "object") {

22
packages/builder/src/components/automation/SetupPanel/AutomationBlockSetup.svelte
View File

@ -14,6 +14,7 @@
notifications,
Checkbox,
DatePicker,
DrawerContent,
} from "@budibase/bbui"
import CreateWebhookModal from "components/automation/Shared/CreateWebhookModal.svelte"
import { automationStore, selectedAutomation, tables } from "stores/builder"
@ -37,7 +38,7 @@
hbAutocomplete,
EditorModes,
} from "components/common/CodeEditor"
import FilterDrawer from "components/design/settings/controls/FilterEditor/FilterDrawer.svelte"
import FilterBuilder from "components/design/settings/controls/FilterEditor/FilterBuilder.svelte"
import { LuceneUtils, Utils } from "@budibase/frontend-core"
import {
getSchemaForDatasourcePlus,
@ -442,15 +443,16 @@
<Button cta slot="buttons" on:click={() => saveFilters(key)}>
Save
</Button>
<FilterDrawer
slot="body"
{filters}
{bindings}
{schemaFields}
datasource={{ type: "table", tableId }}
panel={AutomationBindingPanel}
on:change={e => (tempFilters = e.detail)}
/>
<DrawerContent slot="body">
<FilterBuilder
{filters}
{bindings}
{schemaFields}
datasource={{ type: "table", tableId }}
panel={AutomationBindingPanel}
on:change={e => (tempFilters = e.detail)}
/>
</DrawerContent>
</Drawer>
{:else if value.customType === "password"}
<Input

4
packages/builder/src/components/backend/DataTable/buttons/TableFilterButton.svelte
View File

@ -1,7 +1,7 @@
<script>
import { createEventDispatcher } from "svelte"
import { ActionButton, Modal, ModalContent } from "@budibase/bbui"
import FilterDrawer from "components/design/settings/controls/FilterEditor/FilterDrawer.svelte"
import FilterBuilder from "components/design/settings/controls/FilterEditor/FilterBuilder.svelte"
export let schema
export let filters
@ -40,7 +40,7 @@
onConfirm={() => dispatch("change", tempValue)}
>
<div class="wrapper">
<FilterDrawer
<FilterBuilder
allowBindings={false}
{filters}
{schemaFields}

52
packages/builder/src/components/backend/DataTable/modals/CreateEditColumn.svelte
View File

@ -13,6 +13,7 @@
Layout,
AbsTooltip,
} from "@budibase/bbui"
import { SWITCHABLE_TYPES, ValidColumnNameRegex } from "@budibase/shared-core"
import { createEventDispatcher, getContext, onMount } from "svelte"
import { cloneDeep } from "lodash/fp"
import { tables, datasources } from "stores/builder"
@ -20,11 +21,6 @@
import {
FIELDS,
RelationshipType,
ALLOWABLE_STRING_OPTIONS,
ALLOWABLE_NUMBER_OPTIONS,
ALLOWABLE_STRING_TYPES,
ALLOWABLE_NUMBER_TYPES,
SWITCHABLE_TYPES,
PrettyRelationshipDefinitions,
DB_TYPE_EXTERNAL,
} from "constants/backend"
@ -33,21 +29,20 @@
import ModalBindableInput from "components/common/bindings/ModalBindableInput.svelte"
import { getBindings } from "components/backend/DataTable/formula"
import JSONSchemaModal from "./JSONSchemaModal.svelte"
import { ValidColumnNameRegex } from "@budibase/shared-core"
import { FieldType, FieldSubtype, SourceName } from "@budibase/types"
import RelationshipSelector from "components/common/RelationshipSelector.svelte"
import { RowUtils } from "@budibase/frontend-core"
import ServerBindingPanel from "components/common/bindings/ServerBindingPanel.svelte"
const AUTO_TYPE = FIELDS.AUTO.type
const FORMULA_TYPE = FIELDS.FORMULA.type
const LINK_TYPE = FIELDS.LINK.type
const STRING_TYPE = FIELDS.STRING.type
const NUMBER_TYPE = FIELDS.NUMBER.type
const JSON_TYPE = FIELDS.JSON.type
const DATE_TYPE = FIELDS.DATETIME.type
const USER_TYPE = FIELDS.USER.subtype
const USERS_TYPE = FIELDS.USERS.subtype
const AUTO_TYPE = FieldType.AUTO
const FORMULA_TYPE = FieldType.FORMULA
const LINK_TYPE = FieldType.LINK
const STRING_TYPE = FieldType.STRING
const NUMBER_TYPE = FieldType.NUMBER
const JSON_TYPE = FieldType.JSON
const DATE_TYPE = FieldType.DATETIME
const USER_TYPE = FieldSubtype.USER
const USERS_TYPE = FieldSubtype.USERS
const dispatch = createEventDispatcher()
const PROHIBITED_COLUMN_NAMES = ["type", "_id", "_rev", "tableId"]
@ -61,8 +56,8 @@
let primaryDisplay
let indexes = [...($tables.selected.indexes || [])]
let isCreating = undefined
let relationshipPart1 = PrettyRelationshipDefinitions.Many
let relationshipPart2 = PrettyRelationshipDefinitions.One
let relationshipPart1 = PrettyRelationshipDefinitions.MANY
let relationshipPart2 = PrettyRelationshipDefinitions.ONE
let relationshipTableIdPrimary = null
let relationshipTableIdSecondary = null
let table = $tables.selected
@ -175,7 +170,7 @@
$: typeEnabled =
!originalName ||
(originalName &&
SWITCHABLE_TYPES.indexOf(editableColumn.type) !== -1 &&
SWITCHABLE_TYPES[field.type] &&
!editableColumn?.autocolumn)
const fieldDefinitions = Object.values(FIELDS).reduce(
@ -367,16 +362,15 @@
}
function getAllowedTypes() {
if (
originalName &&
ALLOWABLE_STRING_TYPES.indexOf(editableColumn.type) !== -1
) {
return ALLOWABLE_STRING_OPTIONS
} else if (
originalName &&
ALLOWABLE_NUMBER_TYPES.indexOf(editableColumn.type) !== -1
) {
return ALLOWABLE_NUMBER_OPTIONS
if (originalName) {
const possibleTypes = (
SWITCHABLE_TYPES[field.type] || [editableColumn.type]
).map(t => t.toLowerCase())
return Object.entries(FIELDS)
.filter(([fieldType]) =>
possibleTypes.includes(fieldType.toLowerCase())
)
.map(([_, fieldDefinition]) => fieldDefinition)
}
const isUsers =
@ -632,7 +626,7 @@
/>
</div>
</div>
{:else if editableColumn.type === FieldType.LINK}
{:else if editableColumn.type === FieldType.LINK && !editableColumn.autocolumn}
<RelationshipSelector
bind:relationshipPart1
bind:relationshipPart2

9
packages/builder/src/components/common/Dropzone.svelte
View File

@ -27,14 +27,6 @@
return []
}
}
async function deleteAttachments(fileList) {
try {
return await API.deleteBuilderAttachments(fileList)
} catch (error) {
return []
}
}
</script>
<Dropzone
@ -42,6 +34,5 @@
{label}
{...$$restProps}
{processFiles}
{deleteAttachments}
{handleFileTooLarge}
/>

1
packages/builder/src/components/common/users/PasswordRepeatInput.svelte
View File

@ -9,7 +9,6 @@
"",
requiredValidator
)
// eslint-disable-next-line no-unused-vars
const [repeatPassword, _, repeatTouched] = createValidationStore(
"",
requiredValidator

84
packages/builder/src/components/design/settings/controls/FilterEditor/FilterBuilder.svelte Normal file
View File

@ -0,0 +1,84 @@
<script>
import DrawerBindableInput from "components/common/bindings/DrawerBindableInput.svelte"
import ClientBindingPanel from "components/common/bindings/ClientBindingPanel.svelte"
import { dataFilters } from "@budibase/shared-core"
import { FilterBuilder } from "@budibase/frontend-core"
import { createEventDispatcher, onMount } from "svelte"
export let schemaFields
export let filters = []
export let bindings = []
export let panel = ClientBindingPanel
export let allowBindings = true
export let datasource
const dispatch = createEventDispatcher()
let rawFilters
$: parseFilters(rawFilters)
$: dispatch("change", enrichFilters(rawFilters))
// Remove field key prefixes and determine which behaviours to use
const parseFilters = filters => {
rawFilters = (filters || []).map(filter => {
const { field } = filter
let newFilter = { ...filter }
delete newFilter.allOr
newFilter.field = dataFilters.removeKeyNumbering(field)
return newFilter
})
}
onMount(() => {
parseFilters(filters)
rawFilters.forEach(filter => {
filter.type =
schemaFields.find(field => field.name === filter.field)?.type ||
filter.type
})
})
// Add field key prefixes and a special metadata filter object to indicate
// how to handle filter behaviour
const enrichFilters = rawFilters => {
let count = 1
return rawFilters
.filter(filter => filter.field)
.map(filter => ({
...filter,
field: `${count++}:${filter.field}`,
}))
.concat(...rawFilters.filter(filter => !filter.field))
}
</script>
<FilterBuilder
bind:filters={rawFilters}
behaviourFilters={true}
{schemaFields}
{datasource}
{allowBindings}
>
<div slot="filtering-hero-content" />
<DrawerBindableInput
let:filter
slot="binding"
disabled={filter.noValue}
title={filter.field}
value={filter.value}
placeholder="Value"
{panel}
{bindings}
on:change={event => {
const indexToUpdate = rawFilters.findIndex(f => f.id === filter.id)
rawFilters[indexToUpdate] = {
...rawFilters[indexToUpdate],
value: event.detail,
}
}}
/>
</FilterBuilder>

27
packages/builder/src/components/design/settings/controls/FilterEditor/FilterEditor.svelte
View File

@ -1,8 +1,14 @@
<script>
import { notifications, ActionButton, Button, Drawer } from "@budibase/bbui"
import {
notifications,
ActionButton,
Button,
Drawer,
DrawerContent,
} from "@budibase/bbui"
import { createEventDispatcher } from "svelte"
import { getDatasourceForProvider, getSchemaForDatasource } from "dataBinding"
import FilterDrawer from "./FilterDrawer.svelte"
import FilterBuilder from "./FilterBuilder.svelte"
import { selectedScreen } from "stores/builder"
const dispatch = createEventDispatcher()
@ -40,14 +46,15 @@
</div>
<Drawer bind:this={drawer} title="Filtering" on:drawerHide on:drawerShow>
<Button cta slot="buttons" on:click={saveFilter}>Save</Button>
<FilterDrawer
slot="body"
filters={value}
{bindings}
{schemaFields}
{datasource}
on:change={e => (tempValue = e.detail)}
/>
<DrawerContent slot="body">
<FilterBuilder
filters={value}
{bindings}
{schemaFields}
{datasource}
on:change={e => (tempValue = e.detail)}
/>
</DrawerContent>
</Drawer>
<style>

20
packages/builder/src/constants/backend/index.js
View File

@ -202,26 +202,6 @@ export const PrettyRelationshipDefinitions = {
ONE: "One row",
}
export const ALLOWABLE_STRING_OPTIONS = [
FIELDS.STRING,
FIELDS.OPTIONS,
FIELDS.LONGFORM,
FIELDS.BARCODEQR,
]
export const ALLOWABLE_STRING_TYPES = ALLOWABLE_STRING_OPTIONS.map(
opt => opt.type
)
export const ALLOWABLE_NUMBER_OPTIONS = [FIELDS.NUMBER, FIELDS.BOOLEAN]
export const ALLOWABLE_NUMBER_TYPES = ALLOWABLE_NUMBER_OPTIONS.map(
opt => opt.type
)
export const SWITCHABLE_TYPES = [
...ALLOWABLE_STRING_TYPES,
...ALLOWABLE_NUMBER_TYPES,
]
export const BUDIBASE_INTERNAL_DB_ID = INTERNAL_TABLE_SOURCE_ID
export const DEFAULT_BB_DATASOURCE_ID = "datasource_internal_bb_default"
export const BUDIBASE_DATASOURCE_TYPE = "budibase"

2
packages/builder/src/pages/builder/app/[application]/data/_components/CreateExternalDatasourceModal/stores/datasourceCreation.test.js
View File

@ -4,8 +4,6 @@ import {
createDatasourceCreationStore,
} from "./datasourceCreation"
import { get } from "svelte/store"
// eslint-disable-next-line no-unused-vars
import { shouldIntegrationFetchTableNames } from "stores/selectors"
vi.mock("stores/selectors", () => ({
shouldIntegrationFetchTableNames: vi.fn(),

4
packages/builder/src/pages/builder/app/[application]/data/_components/CreateExternalDatasourceModal/stores/onGoogleAuth.test.js
View File

@ -1,9 +1,9 @@
import { it, expect, describe, beforeEach, vi } from "vitest"
import { createOnGoogleAuthStore } from "./onGoogleAuth"
import { writable, get } from "svelte/store"
// eslint-disable-next-line no-unused-vars
// eslint-disable-next-line
import { params } from "@roxi/routify"
// eslint-disable-next-line no-unused-vars
// eslint-disable-next-line
import { integrations } from "stores/builder"
import { IntegrationTypes } from "constants/backend"

1
packages/builder/src/pages/builder/app/[application]/design/[screenId]/[componentId]/_components/Component/ConditionalUIDrawer.svelte
View File

@ -189,6 +189,7 @@
<Select
options={settingOptions}
bind:value={condition.setting}
on:change={() => delete condition.settingValue}
/>
<div>TO</div>
{#if definition}

10
packages/builder/src/stores/builder/tables.js
View File

@ -1,8 +1,8 @@
import { FieldType } from "@budibase/types"
import { SWITCHABLE_TYPES } from "@budibase/shared-core"
import { get, writable, derived } from "svelte/store"
import { cloneDeep } from "lodash/fp"
import { API } from "api"
import { SWITCHABLE_TYPES } from "constants/backend"
export function createTablesStore() {
const store = writable({
@ -64,7 +64,7 @@ export function createTablesStore() {
if (
oldField != null &&
oldField?.type !== field.type &&
SWITCHABLE_TYPES.indexOf(oldField?.type) === -1
!SWITCHABLE_TYPES[oldField?.type]?.includes(field.type)
) {
updatedTable.schema[key] = oldField
}
@ -148,12 +148,6 @@ export function createTablesStore() {
if (indexes) {
draft.indexes = indexes
}
// Add object to indicate if column is being added
if (draft.schema[field.name] === undefined) {
draft._add = {
name: field.name,
}
}
draft.schema = {
...draft.schema,
[field.name]: cloneDeep(field),

1
packages/cli/src/hosting/utils.ts
View File

@ -105,7 +105,6 @@ export function getAppService(path: string) {
}
export function updateDockerComposeService(
// eslint-disable-next-line no-unused-vars
updateFn: (service: DockerCompose) => void
) {
const opts = ["docker-compose.yaml", "docker-compose.yml"]

240
packages/client/src/components/ClientApp.svelte
View File

@ -119,140 +119,142 @@
{/if}
</svelte:head>
<div
id="spectrum-root"
lang="en"
dir="ltr"
class="spectrum spectrum--medium {$themeStore.baseTheme} {$themeStore.theme}"
class:builder={$builderStore.inBuilder}
class:show={fontsLoaded && dataLoaded}
>
{#if $environmentStore.maintenance.length > 0}
<MaintenanceScreen maintenanceList={$environmentStore.maintenance} />
{:else}
<DeviceBindingsProvider>
<UserBindingsProvider>
<StateBindingsProvider>
<RowSelectionProvider>
<QueryParamsProvider>
<SnippetsProvider>
<!-- Settings bar can be rendered outside of device preview -->
<!-- Key block needs to be outside the if statement or it breaks -->
{#key $builderStore.selectedComponentId}
{#if $builderStore.inBuilder}
<SettingsBar />
{/if}
{/key}
<!-- Clip boundary for selection indicators -->
<div
id="clip-root"
class:preview={$builderStore.inBuilder}
class:tablet-preview={$builderStore.previewDevice ===
"tablet"}
class:mobile-preview={$builderStore.previewDevice ===
"mobile"}
>
<!-- Actual app -->
<div id="app-root">
{#if showDevTools}
<DevToolsHeader />
{#if dataLoaded}
<div
id="spectrum-root"
lang="en"
dir="ltr"
class="spectrum spectrum--medium {$themeStore.baseTheme} {$themeStore.theme}"
class:builder={$builderStore.inBuilder}
class:show={fontsLoaded && dataLoaded}
>
{#if $environmentStore.maintenance.length > 0}
<MaintenanceScreen maintenanceList={$environmentStore.maintenance} />
{:else}
<DeviceBindingsProvider>
<UserBindingsProvider>
<StateBindingsProvider>
<RowSelectionProvider>
<QueryParamsProvider>
<SnippetsProvider>
<!-- Settings bar can be rendered outside of device preview -->
<!-- Key block needs to be outside the if statement or it breaks -->
{#key $builderStore.selectedComponentId}
{#if $builderStore.inBuilder}
<SettingsBar />
{/if}
{/key}
<div id="app-body">
{#if permissionError}
<div class="error">
<Layout justifyItems="center" gap="S">
<!-- eslint-disable-next-line svelte/no-at-html-tags -->
{@html ErrorSVG}
<Heading size="L">
You don't have permission to use this app
</Heading>
<Body size="S">
Ask your administrator to grant you access
</Body>
</Layout>
</div>
{:else if !$screenStore.activeLayout}
<div class="error">
<Layout justifyItems="center" gap="S">
<!-- eslint-disable-next-line svelte/no-at-html-tags -->
{@html ErrorSVG}
<Heading size="L">
Something went wrong rendering your app
</Heading>
<Body size="S">
Get in touch with support if this issue persists
</Body>
</Layout>
</div>
{:else if embedNoScreens}
<div class="error">
<Layout justifyItems="center" gap="S">
<!-- eslint-disable-next-line svelte/no-at-html-tags -->
{@html ErrorSVG}
<Heading size="L">
This Budibase app is not publicly accessible
</Heading>
</Layout>
</div>
{:else}
<CustomThemeWrapper>
{#key $screenStore.activeLayout._id}
<Component
isLayout
instance={$screenStore.activeLayout.props}
/>
{/key}
<!-- Clip boundary for selection indicators -->
<div
id="clip-root"
class:preview={$builderStore.inBuilder}
class:tablet-preview={$builderStore.previewDevice ===
"tablet"}
class:mobile-preview={$builderStore.previewDevice ===
"mobile"}
>
<!-- Actual app -->
<div id="app-root">
{#if showDevTools}
<DevToolsHeader />
{/if}
<!--
<div id="app-body">
{#if permissionError}
<div class="error">
<Layout justifyItems="center" gap="S">
<!-- eslint-disable-next-line svelte/no-at-html-tags -->
{@html ErrorSVG}
<Heading size="L">
You don't have permission to use this app
</Heading>
<Body size="S">
Ask your administrator to grant you access
</Body>
</Layout>
</div>
{:else if !$screenStore.activeLayout}
<div class="error">
<Layout justifyItems="center" gap="S">
<!-- eslint-disable-next-line svelte/no-at-html-tags -->
{@html ErrorSVG}
<Heading size="L">
Something went wrong rendering your app
</Heading>
<Body size="S">
Get in touch with support if this issue persists
</Body>
</Layout>
</div>
{:else if embedNoScreens}
<div class="error">
<Layout justifyItems="center" gap="S">
<!-- eslint-disable-next-line svelte/no-at-html-tags -->
{@html ErrorSVG}
<Heading size="L">
This Budibase app is not publicly accessible
</Heading>
</Layout>
</div>
{:else}
<CustomThemeWrapper>
{#key $screenStore.activeLayout._id}
<Component
isLayout
instance={$screenStore.activeLayout.props}
/>
{/key}
<!--
Flatpickr needs to be inside the theme wrapper.
It also needs its own container because otherwise it hijacks
key events on the whole page. It is painful to work with.
-->
<div id="flatpickr-root" />
<div id="flatpickr-root" />
<!-- Modal container to ensure they sit on top -->
<div class="modal-container" />
<!-- Modal container to ensure they sit on top -->
<div class="modal-container" />
<!-- Layers on top of app -->
<NotificationDisplay />
<ConfirmationDisplay />
<PeekScreenDisplay />
</CustomThemeWrapper>
{/if}
<!-- Layers on top of app -->
<NotificationDisplay />
<ConfirmationDisplay />
<PeekScreenDisplay />
</CustomThemeWrapper>
{/if}
{#if showDevTools}
<DevTools />
{#if showDevTools}
<DevTools />
{/if}
</div>
{#if !$builderStore.inBuilder && $featuresStore.logoEnabled}
<FreeFooter />
{/if}
</div>
{#if !$builderStore.inBuilder && $featuresStore.logoEnabled}
<FreeFooter />
<!-- Preview and dev tools utilities -->
{#if $appStore.isDevApp}
<SelectionIndicator />
{/if}
{#if $builderStore.inBuilder || $devToolsStore.allowSelection}
<HoverIndicator />
{/if}
{#if $builderStore.inBuilder}
<DNDHandler />
<GridDNDHandler />
{/if}
</div>
<!-- Preview and dev tools utilities -->
{#if $appStore.isDevApp}
<SelectionIndicator />
{/if}
{#if $builderStore.inBuilder || $devToolsStore.allowSelection}
<HoverIndicator />
{/if}
{#if $builderStore.inBuilder}
<DNDHandler />
<GridDNDHandler />
{/if}
</div>
</SnippetsProvider>
</QueryParamsProvider>
</RowSelectionProvider>
</StateBindingsProvider>
</UserBindingsProvider>
</DeviceBindingsProvider>
{/if}
</div>
<KeyboardManager />
</SnippetsProvider>
</QueryParamsProvider>
</RowSelectionProvider>
</StateBindingsProvider>
</UserBindingsProvider>
</DeviceBindingsProvider>
{/if}
</div>
<KeyboardManager />
{/if}
<style>
#spectrum-root {

5
packages/client/src/components/app/NavItem.svelte
View File

@ -157,6 +157,11 @@
width: 100%;
}
/* Use normal theme colors for links when using a top nav */
.dropdown:not(.left) .sublinks a {
color: var(--spectrum-alias-text-color);
}
/* Left dropdowns */
.dropdown.left .sublinks-wrapper {
display: none;

216
packages/client/src/components/app/dynamic-filter/FilterModal.svelte
View File

@ -1,216 +1,14 @@
<script>
import {
Body,
Button,
Combobox,
DatePicker,
Icon,
Input,
Layout,
Select,
} from "@budibase/bbui"
import { generate } from "shortid"
import { LuceneUtils, Constants } from "@budibase/frontend-core"
import { getContext } from "svelte"
import { FilterBuilder } from "@budibase/frontend-core"
export let schemaFields
export let filters = []
export let datasource
const context = getContext("context")
const BannedTypes = ["link", "attachment", "json"]
$: fieldOptions = (schemaFields ?? [])
.filter(
field =>
!BannedTypes.includes(field.type) ||
(field.type === "formula" && field.formulaType === "static")
)
.map(field => ({
label: field.displayName || field.name,
value: field.name,
}))
const addFilter = () => {
filters = [
...filters,
{
id: generate(),
field: null,
operator: Constants.OperatorOptions.Equals.value,
value: null,
valueType: "Value",
},
]
}
const removeFilter = id => {
filters = filters.filter(field => field.id !== id)
}
const duplicateFilter = id => {
const existingFilter = filters.find(filter => filter.id === id)
const duplicate = { ...existingFilter, id: generate() }
filters = [...filters, duplicate]
}
const onFieldChange = (expression, field) => {
// Update the field type
expression.type = schemaFields.find(x => x.name === field)?.type
expression.externalType = schemaFields.find(
x => x.name === field
)?.externalType
// Ensure a valid operator is set
const validOperators = LuceneUtils.getValidOperatorsForType(
{ type: expression.type },
expression.field,
datasource
).map(x => x.value)
if (!validOperators.includes(expression.operator)) {
expression.operator =
validOperators[0] ?? Constants.OperatorOptions.Equals.value
onOperatorChange(expression, expression.operator)
}
// if changed to an array, change default value to empty array
const idx = filters.findIndex(x => x.field === field)
if (expression.type === "array") {
filters[idx].value = []
} else {
filters[idx].value = null
}
}
const onOperatorChange = (expression, operator) => {
const noValueOptions = [
Constants.OperatorOptions.Empty.value,
Constants.OperatorOptions.NotEmpty.value,
]
expression.noValue = noValueOptions.includes(operator)
if (expression.noValue) {
expression.value = null
}
}
const getFieldOptions = field => {
const schema = schemaFields.find(x => x.name === field)
return schema?.constraints?.inclusion || []
}
const getSchema = filter => {
return schemaFields.find(field => field.name === filter.field)
}
</script>
<div class="container" class:mobile={$context.device.mobile}>
<Layout noPadding>
<Body size="S">
{#if !filters?.length}
Add your first filter expression.
{:else}
Results are filtered to only those which match all of the following
constraints.
{/if}
</Body>
{#if filters?.length}
<div class="fields">
{#each filters as filter}
<Select
bind:value={filter.field}
options={fieldOptions}
on:change={e => onFieldChange(filter, e.detail)}
placeholder="Column"
/>
<Select
disabled={!filter.field}
options={LuceneUtils.getValidOperatorsForType(
{ type: filter.type, subtype: filter.subtype },
filter.field,
datasource
)}
bind:value={filter.operator}
on:change={e => onOperatorChange(filter, e.detail)}
placeholder={null}
/>
{#if ["string", "longform", "number", "bigint", "formula"].includes(filter.type)}
<Input disabled={filter.noValue} bind:value={filter.value} />
{:else if ["options", "array"].includes(filter.type)}
<Combobox
disabled={filter.noValue}
options={getFieldOptions(filter.field)}
bind:value={filter.value}
/>
{:else if filter.type === "boolean"}
<Combobox
disabled={filter.noValue}
options={[
{ label: "True", value: "true" },
{ label: "False", value: "false" },
]}
bind:value={filter.value}
/>
{:else if filter.type === "datetime"}
<DatePicker
disabled={filter.noValue}
enableTime={!getSchema(filter).dateOnly}
timeOnly={getSchema(filter).timeOnly}
bind:value={filter.value}
/>
{:else}
<Input disabled />
{/if}
<div class="controls">
<Icon
name="Duplicate"
hoverable
size="S"
on:click={() => duplicateFilter(filter.id)}
/>
<Icon
name="Close"
hoverable
size="S"
on:click={() => removeFilter(filter.id)}
/>
</div>
{/each}
</div>
{/if}
<div>
<Button icon="AddCircle" size="M" secondary on:click={addFilter}>
Add filter
</Button>
</div>
</Layout>
</div>
<style>
.container {
width: 100%;
max-width: 1000px;
margin: 0 auto;
}
.fields {
display: grid;
column-gap: var(--spacing-l);
row-gap: var(--spacing-s);
align-items: center;
grid-template-columns: 1fr 120px 1fr auto auto;
}
.controls {
display: contents;
}
.container.mobile .fields {
grid-template-columns: 1fr;
}
.container.mobile .controls {
display: flex;
flex-direction: row;
justify-content: flex-start;
align-items: center;
padding: var(--spacing-s) 0;
gap: var(--spacing-s);
}
</style>
<FilterBuilder bind:filters {schemaFields} {datasource} filtersLabel={null}>
<div slot="filtering-hero-content">
Results are filtered to only those which match all of the following
constraints.
</div>
</FilterBuilder>

12
packages/client/src/components/app/forms/AttachmentField.svelte
View File

@ -58,17 +58,6 @@
}
}
const deleteAttachments = async fileList => {
try {
return await API.deleteAttachments({
keys: fileList,
tableId: formContext?.dataSource?.tableId,
})
} catch (error) {
return []
}
}
const handleChange = e => {
const value = fieldApiMapper.set(e.detail)
const changed = fieldApi.setValue(value)
@ -98,7 +87,6 @@
error={fieldState.error}
on:change={handleChange}
{processFiles}
{deleteAttachments}
{handleFileTooLarge}
{handleTooManyFiles}
{maximum}

1
packages/frontend-core/package.json
View File

@ -11,6 +11,7 @@
"@budibase/types": "0.0.0",
"dayjs": "^1.10.8",
"lodash": "4.17.21",
"shortid": "2.2.15",
"socket.io-client": "^4.6.1"
}
}

28
packages/frontend-core/src/api/attachments.js
View File

@ -61,34 +61,6 @@ export const buildAttachmentEndpoints = API => {
})
return { publicUrl }
},
/**
* Deletes attachments from the bucket.
* @param keys the attachments to delete
* @param tableId the associated table ID
*/
deleteAttachments: async ({ keys, tableId }) => {
return await API.post({
url: `/api/attachments/${tableId}/delete`,
body: {
keys,
},
})
},
/**
* Deletes attachments from the builder bucket.
* @param keys the attachments to delete
*/
deleteBuilderAttachments: async keys => {
return await API.post({
url: `/api/attachments/delete`,
body: {
keys,
},
})
},
/**
* Download an attachment from a row given its column name.
* @param datasourceId the ID of the datasource to download from

361
packages/builder/src/components/design/settings/controls/FilterEditor/FilterDrawer.svelte → packages/frontend-core/src/components/FilterBuilder.svelte
View File

@ -4,33 +4,36 @@
Button,
Combobox,
DatePicker,
DrawerContent,
Icon,
Input,
Label,
Layout,
Multiselect,
Select,
Label,
Multiselect,
} from "@budibase/bbui"
import DrawerBindableInput from "components/common/bindings/DrawerBindableInput.svelte"
import ClientBindingPanel from "components/common/bindings/ClientBindingPanel.svelte"
import { FieldType, SearchFilterOperator } from "@budibase/types"
import { generate } from "shortid"
import { Constants, LuceneUtils } from "@budibase/frontend-core"
import { getFields } from "helpers/searchFields"
import { FieldType } from "@budibase/types"
import { createEventDispatcher, onMount } from "svelte"
import { LuceneUtils, Constants } from "@budibase/frontend-core"
import { getContext } from "svelte"
import FilterUsers from "./FilterUsers.svelte"
const { OperatorOptions } = Constants
export let schemaFields
export let filters = []
export let bindings = []
export let panel = ClientBindingPanel
export let allowBindings = true
export let datasource
export let behaviourFilters = false
export let allowBindings = false
export let filtersLabel = "Filters"
$: matchAny = filters?.find(filter => filter.operator === "allOr") != null
$: onEmptyFilter =
filters?.find(filter => filter.onEmptyFilter)?.onEmptyFilter ?? "all"
$: fieldFilters = filters.filter(
filter => filter.operator !== "allOr" && !filter.onEmptyFilter
)
const dispatch = createEventDispatcher()
const { OperatorOptions } = Constants
const KeyedFieldRegex = /\d[0-9]*:/g
const behaviourOptions = [
{ value: "and", label: "Match all filters" },
{ value: "or", label: "Match any filter" },
@ -40,62 +43,18 @@
{ value: "none", label: "Return no rows" },
]
let rawFilters
let matchAny = false
let onEmptyFilter = "all"
const context = getContext("context")
$: parseFilters(filters)
$: dispatch("change", enrichFilters(rawFilters, matchAny, onEmptyFilter))
$: enrichedSchemaFields = getFields(schemaFields || [], { allowLinks: true })
$: fieldOptions = enrichedSchemaFields.map(field => field.name) || []
$: valueTypeOptions = allowBindings ? ["Value", "Binding"] : ["Value"]
// Remove field key prefixes and determine which behaviours to use
const parseFilters = filters => {
matchAny = filters?.find(filter => filter.operator === "allOr") != null
onEmptyFilter =
filters?.find(filter => filter.onEmptyFilter)?.onEmptyFilter ?? "all"
rawFilters = (filters || [])
.filter(filter => filter.operator !== "allOr" && !filter.onEmptyFilter)
.map(filter => {
const { field } = filter
let newFilter = { ...filter }
delete newFilter.allOr
if (typeof field === "string" && field.match(KeyedFieldRegex) != null) {
const parts = field.split(":")
parts.shift()
newFilter.field = parts.join(":")
}
return newFilter
})
}
onMount(() => {
parseFilters(filters)
rawFilters.forEach(filter => {
filter.type =
schemaFields.find(field => field.name === filter.field)?.type ||
filter.type
})
})
// Add field key prefixes and a special metadata filter object to indicate
// how to handle filter behaviour
const enrichFilters = (rawFilters, matchAny, onEmptyFilter) => {
let count = 1
return rawFilters
.filter(filter => filter.field)
.map(filter => ({
...filter,
field: `${count++}:${filter.field}`,
}))
.concat(matchAny ? [{ operator: "allOr" }] : [])
.concat([{ onEmptyFilter }])
}
$: fieldOptions = (schemaFields ?? [])
.filter(field => getValidOperatorsForType(field).length)
.map(field => ({
label: field.displayName || field.name,
value: field.name,
}))
const addFilter = () => {
rawFilters = [
...rawFilters,
filters = [
...(filters || []),
{
id: generate(),
field: null,
@ -107,22 +66,57 @@
}
const removeFilter = id => {
rawFilters = rawFilters.filter(field => field.id !== id)
filters = filters.filter(field => field.id !== id)
}
const duplicateFilter = id => {
const existingFilter = rawFilters.find(filter => filter.id === id)
const existingFilter = filters.find(filter => filter.id === id)
const duplicate = { ...existingFilter, id: generate() }
rawFilters = [...rawFilters, duplicate]
filters = [...filters, duplicate]
}
const onFieldChange = filter => {
const previousType = filter.type
sanitizeTypes(filter)
sanitizeOperator(filter)
sanitizeValue(filter, previousType)
}
const onOperatorChange = filter => {
sanitizeOperator(filter)
sanitizeValue(filter, filter.type)
}
const onValueTypeChange = filter => {
sanitizeValue(filter)
}
const getFieldOptions = field => {
const schema = schemaFields.find(x => x.name === field)
return schema?.constraints?.inclusion || []
}
const getSchema = filter => {
return enrichedSchemaFields.find(field => field.name === filter.field)
return schemaFields.find(field => field.name === filter.field)
}
const getValidOperatorsForType = filter => {
if (!filter?.field && !filter?.name) {
return []
}
return LuceneUtils.getValidOperatorsForType(
filter,
filter.field || filter.name,
datasource
)
}
$: valueTypeOptions = allowBindings ? ["Value", "Binding"] : ["Value"]
const sanitizeTypes = filter => {
// Update type based on field
const fieldSchema = enrichedSchemaFields.find(x => x.name === filter.field)
const fieldSchema = schemaFields.find(x => x.name === filter.field)
filter.type = fieldSchema?.type
filter.subtype = fieldSchema?.subtype
@ -154,88 +148,79 @@
// Ensure array values are properly set and cleared
if (Array.isArray(filter.value)) {
if (filter.valueType !== "Value" || filter.type !== "array") {
if (filter.valueType !== "Value" || filter.type !== FieldType.ARRAY) {
filter.value = null
}
} else if (filter.type === "array" && filter.valueType === "Value") {
} else if (
filter.type === FieldType.ARRAY &&
filter.valueType === "Value"
) {
filter.value = []
} else if (
previousType !== filter.type &&
(previousType === FieldType.BB_REFERENCE ||
filter.type === FieldType.BB_REFERENCE)
) {
filter.value = filter.type === "array" ? [] : null
filter.value = filter.type === FieldType.ARRAY ? [] : null
}
}
const onFieldChange = filter => {
const previousType = filter.type
sanitizeTypes(filter)
sanitizeOperator(filter)
sanitizeValue(filter, previousType)
}
const onOperatorChange = filter => {
sanitizeOperator(filter)
sanitizeValue(filter, filter.type)
}
const onValueTypeChange = filter => {
sanitizeValue(filter)
}
const getFieldOptions = field => {
const schema = enrichedSchemaFields.find(x => x.name === field)
return schema?.constraints?.inclusion || []
}
const getValidOperatorsForType = filter => {
if (!filter?.field) {
return []
function handleAllOr(option) {
filters = filters.filter(f => f.operator !== "allOr")
if (option === "or") {
filters.push({ operator: "allOr" })
}
}
return LuceneUtils.getValidOperatorsForType(
{ type: filter.type, subtype: filter.subtype },
filter.field,
datasource
)
function handleOnEmptyFilter(value) {
filters = filters?.filter(filter => !filter.onEmptyFilter)
filters.push({ onEmptyFilter: value })
}
</script>
<DrawerContent>
<div class="container">
<Layout noPadding>
{#if !rawFilters?.length}
<Body size="S">Add your first filter expression.</Body>
{:else}
<div class="fields">
<Select
label="Behaviour"
value={matchAny ? "or" : "and"}
options={behaviourOptions}
getOptionLabel={opt => opt.label}
getOptionValue={opt => opt.value}
on:change={e => (matchAny = e.detail === "or")}
placeholder={null}
/>
{#if datasource?.type === "table"}
<Select
label="When filter empty"
value={onEmptyFilter}
options={onEmptyOptions}
getOptionLabel={opt => opt.label}
getOptionValue={opt => opt.value}
on:change={e => (onEmptyFilter = e.detail)}
placeholder={null}
/>
<div class="container" class:mobile={$context?.device?.mobile}>
<Layout noPadding>
{#if fieldOptions?.length}
<Body size="S">
{#if !fieldFilters?.length}
Add your first filter expression.
{:else}
<slot name="filtering-hero-content" />
{#if behaviourFilters}
<div class="behaviour-filters">
<Select
label="Behaviour"
value={matchAny ? "or" : "and"}
options={behaviourOptions}
getOptionLabel={opt => opt.label}
getOptionValue={opt => opt.value}
on:change={e => handleAllOr(e.detail)}
placeholder={null}
/>
{#if datasource?.type === "table"}
<Select
label="When filter empty"
value={onEmptyFilter}
options={onEmptyOptions}
getOptionLabel={opt => opt.label}
getOptionValue={opt => opt.value}
on:change={e => handleOnEmptyFilter(e.detail)}
placeholder={null}
/>
{/if}
</div>
{/if}
</div>
{/if}
</Body>
{#if fieldFilters?.length}
<div>
<div class="filter-label">
<Label>Filters</Label>
</div>
<div class="fields">
{#each rawFilters as filter}
{#if filtersLabel}
<div class="filter-label">
<Label>{filtersLabel}</Label>
</div>
{/if}
<div class="fields" class:with-bindings={allowBindings}>
{#each fieldFilters as filter}
<Select
bind:value={filter.field}
options={fieldOptions}
@ -249,38 +234,32 @@
on:change={() => onOperatorChange(filter)}
placeholder={null}
/>
<Select
disabled={filter.noValue || !filter.field}
options={valueTypeOptions}
bind:value={filter.valueType}
on:change={() => onValueTypeChange(filter)}
placeholder={null}
/>
{#if filter.field && filter.valueType === "Binding"}
<DrawerBindableInput
disabled={filter.noValue}
title={filter.field}
value={filter.value}
placeholder="Value"
{panel}
{bindings}
on:change={event => (filter.value = event.detail)}
{#if allowBindings}
<Select
disabled={filter.noValue || !filter.field}
options={valueTypeOptions}
bind:value={filter.valueType}
on:change={() => onValueTypeChange(filter)}
placeholder={null}
/>
{:else if ["string", "longform", "number", "bigint", "formula"].includes(filter.type)}
{/if}
{#if allowBindings && filter.field && filter.valueType === "Binding"}
<slot name="binding" {filter} />
{:else if [FieldType.STRING, FieldType.LONGFORM, FieldType.NUMBER, FieldType.BIGINT, FieldType.FORMULA].includes(filter.type)}
<Input disabled={filter.noValue} bind:value={filter.value} />
{:else if filter.type === "array" || (filter.type === "options" && filter.operator === "oneOf")}
{:else if filter.type === FieldType.ARRAY || (filter.type === FieldType.OPTIONS && filter.operator === SearchFilterOperator.ONE_OF)}
<Multiselect
disabled={filter.noValue}
options={getFieldOptions(filter.field)}
bind:value={filter.value}
/>
{:else if filter.type === "options"}
{:else if filter.type === FieldType.OPTIONS}
<Combobox
disabled={filter.noValue}
options={getFieldOptions(filter.field)}
bind:value={filter.value}
/>
{:else if filter.type === "boolean"}
{:else if filter.type === FieldType.BOOLEAN}
<Combobox
disabled={filter.noValue}
options={[
@ -289,7 +268,7 @@
]}
bind:value={filter.value}
/>
{:else if filter.type === "datetime"}
{:else if filter.type === FieldType.DATETIME}
<DatePicker
disabled={filter.noValue}
enableTime={!getSchema(filter)?.dateOnly}
@ -306,32 +285,36 @@
disabled={filter.noValue}
/>
{:else}
<DrawerBindableInput disabled />
<Input disabled />
{/if}
<Icon
name="Duplicate"
hoverable
size="S"
on:click={() => duplicateFilter(filter.id)}
/>
<Icon
name="Close"
hoverable
size="S"
on:click={() => removeFilter(filter.id)}
/>
<div class="controls">
<Icon
name="Duplicate"
hoverable
size="S"
on:click={() => duplicateFilter(filter.id)}
/>
<Icon
name="Close"
hoverable
size="S"
on:click={() => removeFilter(filter.id)}
/>
</div>
{/each}
</div>
</div>
{/if}
<div class="bottom">
<div>
<Button icon="AddCircle" size="M" secondary on:click={addFilter}>
Add filter
</Button>
</div>
</Layout>
</div>
</DrawerContent>
{:else}
<Body size="S">None of the table column can be used for filtering.</Body>
{/if}
</Layout>
</div>
<style>
.container {
@ -339,22 +322,42 @@
max-width: 1000px;
margin: 0 auto;
}
.fields {
display: grid;
column-gap: var(--spacing-l);
row-gap: var(--spacing-s);
align-items: center;
grid-template-columns: 1fr 120px 1fr auto auto;
}
.fields.with-bindings {
grid-template-columns: minmax(150px, 1fr) 170px 120px minmax(150px, 1fr) 16px 16px;
}
.controls {
display: contents;
}
.container.mobile .fields {
grid-template-columns: 1fr;
}
.container.mobile .controls {
display: flex;
flex-direction: row;
justify-content: flex-start;
align-items: center;
padding: var(--spacing-s) 0;
gap: var(--spacing-s);
}
.filter-label {
margin-bottom: var(--spacing-s);
}
.bottom {
display: flex;
justify-content: space-between;
.behaviour-filters {
display: grid;
column-gap: var(--spacing-l);
row-gap: var(--spacing-s);
align-items: center;
grid-template-columns: minmax(150px, 1fr) 170px 120px minmax(150px, 1fr) 16px 16px;
}
</style>

4
packages/builder/src/components/design/settings/controls/FilterEditor/FilterUsers.svelte → packages/frontend-core/src/components/FilterUsers.svelte
View File

@ -1,9 +1,9 @@
<script>
import { Select, Multiselect } from "@budibase/bbui"
import { fetchData } from "@budibase/frontend-core"
import { createAPIClient } from "../api"
import { API } from "api"
export let API = createAPIClient()
export let value = null
export let disabled
export let multiselect = false

9
packages/frontend-core/src/components/grid/cells/AttachmentCell.svelte
View File

@ -61,14 +61,6 @@
}
}
const deleteAttachments = async fileList => {
try {
return await API.deleteBuilderAttachments(fileList)
} catch (error) {
return []
}
}
onMount(() => {
api = {
focus: () => open(),
@ -101,7 +93,6 @@
on:change={e => onChange(e.detail)}
maximum={maximum || schema.constraints?.length?.maximum}
{processFiles}
{deleteAttachments}
{handleFileTooLarge}
/>
</div>

1
packages/frontend-core/src/components/index.js
View File

@ -6,3 +6,4 @@ export { default as UserAvatars } from "./UserAvatars.svelte"
export { default as Updating } from "./Updating.svelte"
export { Grid } from "./grid"
export { default as ClientAppSkeleton } from "./ClientAppSkeleton.svelte"
export { default as FilterBuilder } from "./FilterBuilder.svelte"

3
packages/frontend-core/src/fetch/DataFetch.js
View File

@ -348,8 +348,7 @@ export default class DataFetch {
* Determine the feature flag for this datasource definition
* @param definition
*/
// eslint-disable-next-line no-unused-vars
determineFeatureFlags(definition) {
determineFeatureFlags(_definition) {
return {
supportsSearch: false,
supportsSort: false,

2
packages/pro

@ -1 +1 @@
Subproject commit ef186d00241f96037f9fd34d7a3826041977ab3a
Subproject commit 06b1064f7e2f7cac5d4bef2ee999796a2a1f0f2c

2
packages/server/package.json
View File

@ -125,7 +125,7 @@
"@babel/preset-env": "7.16.11",
"@swc/core": "1.3.71",
"@swc/jest": "0.2.27",
"@types/archiver": "^6.0.2",
"@types/archiver": "6.0.2",
"@types/global-agent": "2.1.1",
"@types/google-spreadsheet": "3.1.5",
"@types/jest": "29.5.5",

1
packages/server/scripts/integrations/mssql/docker-compose.yaml
View File

@ -4,6 +4,7 @@ services:
# user: sa
# database: master
mssql:
# platform: linux/amd64
image: bb/mssql
build:
context: .

2
packages/server/specs/generate.ts
View File

@ -76,7 +76,7 @@ function writeFile(output: any, filename: string) {
console.log(`Wrote spec to ${path}`)
return path
} catch (err) {
console.error(err)
console.error("Error writing spec file", err)
}
}

1
packages/server/src/api/controllers/application.ts
View File

@ -320,6 +320,7 @@ async function performAppCreate(ctx: UserCtx<CreateAppRequest, App>) {
"theme",
"customTheme",
"icon",
"snippets",
]
keys.forEach(key => {
if (existing[key]) {

42
packages/server/src/api/controllers/row/ExternalRequest.ts
View File

@ -36,7 +36,6 @@ import { getDatasourceAndQuery } from "../../../sdk/app/rows/utils"
import { processObjectSync } from "@budibase/string-templates"
import { cloneDeep } from "lodash/fp"
import { db as dbCore } from "@budibase/backend-core"
import AliasTables from "./alias"
import sdk from "../../../sdk"
import env from "../../../environment"
@ -120,6 +119,9 @@ async function removeManyToManyRelationships(
endpoint: getEndpoint(tableId, Operation.DELETE),
body: { [colName]: null },
filters,
meta: {
table,
},
})
} else {
return []
@ -134,6 +136,9 @@ async function removeOneToManyRelationships(rowId: string, table: Table) {
return getDatasourceAndQuery({
endpoint: getEndpoint(tableId, Operation.UPDATE),
filters,
meta: {
table,
},
})
} else {
return []
@ -249,6 +254,9 @@ export class ExternalRequest<T extends Operation> {
const response = await getDatasourceAndQuery({
endpoint: getEndpoint(table._id!, Operation.READ),
filters: buildFilters(rowId, {}, table),
meta: {
table,
},
})
if (Array.isArray(response) && response.length > 0) {
return response[0]
@ -366,36 +374,45 @@ export class ExternalRequest<T extends Operation> {
) {
continue
}
let tableId: string | undefined,
let relatedTableId: string | undefined,
lookupField: string | undefined,
fieldName: string | undefined
if (isManyToMany(field)) {
tableId = field.through
relatedTableId = field.through
lookupField = primaryKey
fieldName = field.throughTo || primaryKey
} else if (isManyToOne(field)) {
tableId = field.tableId
relatedTableId = field.tableId
lookupField = field.foreignKey
fieldName = field.fieldName
}
if (!tableId || !lookupField || !fieldName) {
if (!relatedTableId || !lookupField || !fieldName) {
throw new Error(
"Unable to lookup relationships - undefined column properties."
)
}
const { tableName: relatedTableName } = breakExternalTableId(tableId)
const { tableName: relatedTableName } =
breakExternalTableId(relatedTableId)
// @ts-ignore
const linkPrimaryKey = this.tables[relatedTableName].primary[0]
if (!lookupField || !row[lookupField]) {
continue
}
const endpoint = getEndpoint(relatedTableId, Operation.READ)
const relatedTable = this.tables[endpoint.entityId]
if (!relatedTable) {
throw new Error("unable to find related table")
}
const response = await getDatasourceAndQuery({
endpoint: getEndpoint(tableId, Operation.READ),
endpoint: endpoint,
filters: {
equal: {
[fieldName]: row[lookupField],
},
},
meta: {
table: relatedTable,
},
})
// this is the response from knex if no rows found
const rows: Row[] =
@ -403,7 +420,11 @@ export class ExternalRequest<T extends Operation> {
const storeTo = isManyToMany(field)
? field.throughFrom || linkPrimaryKey
: fieldName
related[storeTo] = { rows, isMany: isManyToMany(field), tableId }
related[storeTo] = {
rows,
isMany: isManyToMany(field),
tableId: relatedTableId,
}
}
return related
}
@ -471,6 +492,9 @@ export class ExternalRequest<T extends Operation> {
// if we're doing many relationships then we're writing, only one response
body,
filters: buildFilters(id, {}, linkTable),
meta: {
table: linkTable,
},
})
)
} else {
@ -618,7 +642,7 @@ export class ExternalRequest<T extends Operation> {
if (env.SQL_ALIASING_DISABLE) {
response = await getDatasourceAndQuery(json)
} else {
const aliasing = new AliasTables(Object.keys(this.tables))
const aliasing = new sdk.rows.AliasTables(Object.keys(this.tables))
response = await aliasing.queryWithAliasing(json)
}

9
packages/server/src/api/controllers/row/utils/basic.ts
View File

@ -62,12 +62,12 @@ export function basicProcessing({
row,
table,
isLinked,
internal,
sqs,
}: {
row: Row
table: Table
isLinked: boolean
internal?: boolean
sqs?: boolean
}): Row {
const thisRow: Row = {}
// filter the row down to what is actually the row (not joined)
@ -84,12 +84,13 @@ export function basicProcessing({
thisRow[fieldName] = value
}
}
if (!internal) {
if (!sqs) {
thisRow._id = generateIdForRow(row, table, isLinked)
thisRow.tableId = table._id
thisRow._rev = "rev"
} else {
for (let internalColumn of CONSTANT_INTERNAL_ROW_COLS) {
const columns = Object.keys(table.schema)
for (let internalColumn of [...CONSTANT_INTERNAL_ROW_COLS, ...columns]) {
thisRow[internalColumn] = extractFieldValue({
row,
tableName: table._id!,

4
packages/server/src/api/controllers/row/utils/sqlUtils.ts
View File

@ -51,11 +51,11 @@ export async function updateRelationshipColumns(
continue
}
let linked = await basicProcessing({
let linked = basicProcessing({
row,
table: linkedTable,
isLinked: true,
internal: opts?.sqs,
sqs: opts?.sqs,
})
if (!linked._id) {
continue

6
packages/server/src/api/controllers/row/utils/utils.ts
View File

@ -132,6 +132,7 @@ export async function sqlOutputProcessing(
let rowId = row._id
if (opts?.sqs) {
rowId = getInternalRowId(row, table)
row._id = rowId
} else if (!rowId) {
rowId = generateIdForRow(row, table)
row._id = rowId
@ -153,7 +154,7 @@ export async function sqlOutputProcessing(
row,
table,
isLinked: false,
internal: opts?.sqs,
sqs: opts?.sqs,
}),
table
)
@ -167,7 +168,8 @@ export async function sqlOutputProcessing(
tables,
row,
finalRows,
relationships
relationships,
opts
)
}

7
packages/server/src/api/controllers/static/index.ts
View File

@ -127,13 +127,6 @@ export const uploadFile = async function (
)
}
export const deleteObjects = async function (ctx: Ctx) {
ctx.body = await objectStore.deleteFiles(
ObjectStoreBuckets.APPS,
ctx.request.body.keys
)
}
const requiresMigration = async (ctx: Ctx) => {
const appId = context.getAppId()
if (!appId) {

1
packages/server/src/api/controllers/table/ExternalRequest.ts
View File

@ -22,6 +22,7 @@ export async function makeTableRequest(
operation,
},
meta: {
table,
tables,
},
table,

3
packages/server/src/api/controllers/table/external.ts
View File

@ -31,7 +31,6 @@ export async function save(
renaming?: RenameColumn
) {
const inputs = ctx.request.body
const adding = inputs?._add
// can't do this right now
delete inputs.rows
const tableId = ctx.request.body._id
@ -44,7 +43,7 @@ export async function save(
const { datasource, table } = await sdk.tables.external.save(
datasourceId!,
inputs,
{ tableId, renaming, adding }
{ tableId, renaming }
)
builderSocket?.emitDatasourceUpdate(ctx, datasource)
return table

5
packages/server/src/api/controllers/table/index.ts
View File

@ -77,11 +77,6 @@ export async function save(ctx: UserCtx<SaveTableRequest, SaveTableResponse>) {
const renaming = ctx.request.body._rename
const api = pickApi({ table })
// do not pass _rename or _add if saving to CouchDB
if (api === internal) {
delete ctx.request.body._add
delete ctx.request.body._rename
}
let savedTable = await api.save(ctx, renaming)
if (!table._id) {
savedTable = sdk.tables.enrichViewSchemas(savedTable)

2
packages/server/src/api/controllers/table/internal.ts
View File

@ -16,7 +16,7 @@ export async function save(
ctx: UserCtx<SaveTableRequest, SaveTableResponse>,
renaming?: RenameColumn
) {
const { rows, ...rest } = ctx.request.body
const { _rename, rows, ...rest } = ctx.request.body
let tableToSave: Table = {
_id: generateTableID(),
...rest,

11
packages/server/src/api/routes/static.ts
View File

@ -32,11 +32,6 @@ router
.get("/builder/:file*", controller.serveBuilder)
.get("/api/assets/client", controller.serveClientLibrary)
.post("/api/attachments/process", authorized(BUILDER), controller.uploadFile)
.post(
"/api/attachments/delete",
authorized(BUILDER),
controller.deleteObjects
)
.post("/api/beta/:feature", controller.toggleBetaUiFeature)
.post(
"/api/attachments/:tableId/upload",
@ -44,12 +39,6 @@ router
authorized(PermissionType.TABLE, PermissionLevel.WRITE),
controller.uploadFile
)
.post(
"/api/attachments/:tableId/delete",
paramResource("tableId"),
authorized(PermissionType.TABLE, PermissionLevel.WRITE),
controller.deleteObjects
)
.get("/app/preview", authorized(BUILDER), controller.serveBuilderPreview)
.get("/app/:appUrl/:path*", controller.serveApp)
.get("/:appId/:path*", controller.serveApp)

162
packages/server/src/api/routes/tests/datasource.spec.ts
View File

@ -6,7 +6,19 @@ import sdk from "../../../sdk"
import tk from "timekeeper"
import { mocks } from "@budibase/backend-core/tests"
import { QueryPreview, SourceName } from "@budibase/types"
import {
Datasource,
FieldSchema,
FieldSubtype,
FieldType,
QueryPreview,
RelationshipType,
SourceName,
Table,
TableSchema,
} from "@budibase/types"
import { DatabaseName, getDatasource } from "../../../integrations/tests/utils"
import { tableForDatasource } from "../../../tests/utilities/structures"
tk.freeze(mocks.date.MOCK_DATE)
@ -223,4 +235,152 @@ describe("/datasources", () => {
})
})
})
describe.each([
[DatabaseName.POSTGRES, getDatasource(DatabaseName.POSTGRES)],
[DatabaseName.MYSQL, getDatasource(DatabaseName.MYSQL)],
[DatabaseName.SQL_SERVER, getDatasource(DatabaseName.SQL_SERVER)],
[DatabaseName.MARIADB, getDatasource(DatabaseName.MARIADB)],
])("fetch schema (%s)", (_, dsProvider) => {
beforeAll(async () => {
datasource = await config.api.datasource.create(await dsProvider)
})
it("fetching schema will not drop tables or columns", async () => {
const datasourceId = datasource!._id!
const simpleTable = await config.api.table.save(
tableForDatasource(datasource, {
name: "simple",
schema: {
name: {
name: "name",
type: FieldType.STRING,
},
},
})
)
type SupportedSqlTypes =
| FieldType.STRING
| FieldType.BARCODEQR
| FieldType.LONGFORM
| FieldType.OPTIONS
| FieldType.DATETIME
| FieldType.NUMBER
| FieldType.BOOLEAN
| FieldType.FORMULA
| FieldType.BIGINT
| FieldType.BB_REFERENCE
| FieldType.LINK
| FieldType.ARRAY
const fullSchema: {
[type in SupportedSqlTypes]: FieldSchema & { type: type }
} = {
[FieldType.STRING]: {
name: "string",
type: FieldType.STRING,
constraints: {
presence: true,
},
},
[FieldType.LONGFORM]: {
name: "longform",
type: FieldType.LONGFORM,
},
[FieldType.OPTIONS]: {
name: "options",
type: FieldType.OPTIONS,
constraints: {
presence: { allowEmpty: false },
},
},
[FieldType.NUMBER]: {
name: "number",
type: FieldType.NUMBER,
},
[FieldType.BOOLEAN]: {
name: "boolean",
type: FieldType.BOOLEAN,
},
[FieldType.ARRAY]: {
name: "array",
type: FieldType.ARRAY,
},
[FieldType.DATETIME]: {
name: "datetime",
type: FieldType.DATETIME,
dateOnly: true,
timeOnly: false,
},
[FieldType.LINK]: {
name: "link",
type: FieldType.LINK,
tableId: simpleTable._id!,
relationshipType: RelationshipType.ONE_TO_MANY,
fieldName: "link",
},
[FieldType.FORMULA]: {
name: "formula",
type: FieldType.FORMULA,
formula: "any formula",
},
[FieldType.BARCODEQR]: {
name: "barcodeqr",
type: FieldType.BARCODEQR,
},
[FieldType.BIGINT]: {
name: "bigint",
type: FieldType.BIGINT,
},
[FieldType.BB_REFERENCE]: {
name: "bb_reference",
type: FieldType.BB_REFERENCE,
subtype: FieldSubtype.USERS,
},
}
await config.api.table.save(
tableForDatasource(datasource, {
name: "full",
schema: fullSchema,
})
)
const persisted = await config.api.datasource.get(datasourceId)
await config.api.datasource.fetchSchema(datasourceId)
const updated = await config.api.datasource.get(datasourceId)
const expected: Datasource = {
...persisted,
entities:
persisted?.entities &&
Object.entries(persisted.entities).reduce<Record<string, Table>>(
(acc, [tableName, table]) => {
acc[tableName] = {
...table,
primaryDisplay: expect.not.stringMatching(
new RegExp(`^${table.primaryDisplay || ""}$`)
),
schema: Object.entries(table.schema).reduce<TableSchema>(
(acc, [fieldName, field]) => {
acc[fieldName] = expect.objectContaining({
...field,
})
return acc
},
{}
),
}
return acc
},
{}
),
_rev: expect.any(String),
}
expect(updated).toEqual(expected)
})
})
})

18
packages/server/src/api/routes/tests/queries/generic-sql.spec.ts
View File

@ -4,6 +4,7 @@ import {
Query,
QueryPreview,
SourceName,
TableSourceType,
} from "@budibase/types"
import * as setup from "../utilities"
import {
@ -740,12 +741,25 @@ describe.each(
})
describe("query through datasource", () => {
it("should be able to query a pg datasource", async () => {
it("should be able to query the datasource", async () => {
const entityId = "test_table"
await config.api.datasource.update({
...datasource,
entities: {
[entityId]: {
name: entityId,
schema: {},
type: "table",
sourceId: datasource._id!,
sourceType: TableSourceType.EXTERNAL,
},
},
})
const res = await config.api.datasource.query({
endpoint: {
datasourceId: datasource._id!,
operation: Operation.READ,
entityId: "test_table",
entityId,
},
resource: {
fields: ["id", "name"],

199
packages/server/src/api/routes/tests/search.spec.ts
View File

@ -26,6 +26,7 @@ describe.each([
[DatabaseName.MARIADB, getDatasource(DatabaseName.MARIADB)],
])("/api/:sourceId/search (%s)", (name, dsProvider) => {
const isSqs = name === "internal-sqs"
const isInternal = name === "internal"
const config = setup.getConfig()
let envCleanup: (() => void) | undefined
@ -336,6 +337,20 @@ describe.each([
expectQuery({
range: { age: { low: 5, high: 9 } },
}).toFindNothing())
// We never implemented half-open ranges in Lucene.
!isInternal &&
it("can search using just a low value", () =>
expectQuery({
range: { age: { low: 5 } },
}).toContainExactly([{ age: 10 }]))
// We never implemented half-open ranges in Lucene.
!isInternal &&
it("can search using just a high value", () =>
expectQuery({
range: { age: { high: 5 } },
}).toContainExactly([{ age: 1 }]))
})
describe("sort", () => {
@ -440,6 +455,20 @@ describe.each([
expectQuery({
range: { dob: { low: JAN_5TH, high: JAN_9TH } },
}).toFindNothing())
// We never implemented half-open ranges in Lucene.
!isInternal &&
it("can search using just a low value", () =>
expectQuery({
range: { dob: { low: JAN_5TH } },
}).toContainExactly([{ dob: JAN_10TH }]))
// We never implemented half-open ranges in Lucene.
!isInternal &&
it("can search using just a high value", () =>
expectQuery({
range: { dob: { high: JAN_5TH } },
}).toContainExactly([{ dob: JAN_1ST }]))
})
describe("sort", () => {
@ -476,4 +505,174 @@ describe.each([
})
})
})
describe("array of strings", () => {
beforeAll(async () => {
await createTable({
numbers: {
name: "numbers",
type: FieldType.ARRAY,
constraints: { inclusion: ["one", "two", "three"] },
},
})
await createRows([{ numbers: ["one", "two"] }, { numbers: ["three"] }])
})
describe("contains", () => {
it("successfully finds a row", () =>
expectQuery({ contains: { numbers: ["one"] } }).toContainExactly([
{ numbers: ["one", "two"] },
]))
it("fails to find nonexistent row", () =>
expectQuery({ contains: { numbers: ["none"] } }).toFindNothing())
it("fails to find row containing all", () =>
expectQuery({
contains: { numbers: ["one", "two", "three"] },
}).toFindNothing())
it("finds all with empty list", () =>
expectQuery({ contains: { numbers: [] } }).toContainExactly([
{ numbers: ["one", "two"] },
{ numbers: ["three"] },
]))
})
describe("notContains", () => {
it("successfully finds a row", () =>
expectQuery({ notContains: { numbers: ["one"] } }).toContainExactly([
{ numbers: ["three"] },
]))
it("fails to find nonexistent row", () =>
expectQuery({
notContains: { numbers: ["one", "two", "three"] },
}).toContainExactly([
{ numbers: ["one", "two"] },
{ numbers: ["three"] },
]))
it("finds all with empty list", () =>
expectQuery({ notContains: { numbers: [] } }).toContainExactly([
{ numbers: ["one", "two"] },
{ numbers: ["three"] },
]))
})
describe("containsAny", () => {
it("successfully finds rows", () =>
expectQuery({
containsAny: { numbers: ["one", "two", "three"] },
}).toContainExactly([
{ numbers: ["one", "two"] },
{ numbers: ["three"] },
]))
it("fails to find nonexistent row", () =>
expectQuery({ containsAny: { numbers: ["none"] } }).toFindNothing())
it("finds all with empty list", () =>
expectQuery({ containsAny: { numbers: [] } }).toContainExactly([
{ numbers: ["one", "two"] },
{ numbers: ["three"] },
]))
})
})
describe("bigints", () => {
const SMALL = "1"
const MEDIUM = "10000000"
// Our bigints are int64s in most datasources.
const BIG = "9223372036854775807"
beforeAll(async () => {
await createTable({
num: { name: "num", type: FieldType.BIGINT },
})
await createRows([{ num: SMALL }, { num: MEDIUM }, { num: BIG }])
})
describe("equal", () => {
it("successfully finds a row", () =>
expectQuery({ equal: { num: SMALL } }).toContainExactly([
{ num: SMALL },
]))
it("successfully finds a big value", () =>
expectQuery({ equal: { num: BIG } }).toContainExactly([{ num: BIG }]))
it("fails to find nonexistent row", () =>
expectQuery({ equal: { num: "2" } }).toFindNothing())
})
describe("notEqual", () => {
it("successfully finds a row", () =>
expectQuery({ notEqual: { num: SMALL } }).toContainExactly([
{ num: MEDIUM },
{ num: BIG },
]))
it("fails to find nonexistent row", () =>
expectQuery({ notEqual: { num: 10 } }).toContainExactly([
{ num: SMALL },
{ num: MEDIUM },
{ num: BIG },
]))
})
describe("oneOf", () => {
it("successfully finds a row", () =>
expectQuery({ oneOf: { num: [SMALL] } }).toContainExactly([
{ num: SMALL },
]))
it("successfully finds all rows", () =>
expectQuery({ oneOf: { num: [SMALL, MEDIUM, BIG] } }).toContainExactly([
{ num: SMALL },
{ num: MEDIUM },
{ num: BIG },
]))
it("fails to find nonexistent row", () =>
expectQuery({ oneOf: { num: [2] } }).toFindNothing())
})
// Range searches against bigints don't seem to work at all in Lucene, and I
// couldn't figure out why. Given that we're replacing Lucene with SQS,
// we've decided not to spend time on it.
!isInternal &&
describe("range", () => {
it("successfully finds a row", () =>
expectQuery({
range: { num: { low: SMALL, high: "5" } },
}).toContainExactly([{ num: SMALL }]))
it("successfully finds multiple rows", () =>
expectQuery({
range: { num: { low: SMALL, high: MEDIUM } },
}).toContainExactly([{ num: SMALL }, { num: MEDIUM }]))
it("successfully finds a row with a high bound", () =>
expectQuery({
range: { num: { low: MEDIUM, high: BIG } },
}).toContainExactly([{ num: MEDIUM }, { num: BIG }]))
it("successfully finds no rows", () =>
expectQuery({
range: { num: { low: "5", high: "5" } },
}).toFindNothing())
it("can search using just a low value", () =>
expectQuery({
range: { num: { low: MEDIUM } },
}).toContainExactly([{ num: MEDIUM }, { num: BIG }]))
it("can search using just a high value", () =>
expectQuery({
range: { num: { high: MEDIUM } },
}).toContainExactly([{ num: SMALL }, { num: MEDIUM }]))
})
})
})

9
packages/server/src/api/routes/tests/table.spec.ts
View File

@ -34,7 +34,7 @@ describe.each([
[DatabaseName.SQL_SERVER, getDatasource(DatabaseName.SQL_SERVER)],
[DatabaseName.MARIADB, getDatasource(DatabaseName.MARIADB)],
])("/tables (%s)", (_, dsProvider) => {
let isInternal: boolean
const isInternal: boolean = !dsProvider
let datasource: Datasource | undefined
let config = setup.getConfig()
@ -44,9 +44,6 @@ describe.each([
await config.init()
if (dsProvider) {
datasource = await config.api.datasource.create(await dsProvider)
isInternal = false
} else {
isInternal = true
}
})
@ -219,9 +216,6 @@ describe.each([
it("should add a new column for an internal DB table", async () => {
const saveTableRequest: SaveTableRequest = {
_add: {
name: "NEW_COLUMN",
},
...basicTable(),
}
@ -235,7 +229,6 @@ describe.each([
updatedAt: expect.stringMatching(ISO_REGEX_PATTERN),
views: {},
}
delete expectedResponse._add
expect(response).toEqual(expectedResponse)
})
})

12
packages/server/src/api/routes/tests/viewV2.spec.ts
View File

@ -8,7 +8,7 @@ import {
PermissionLevel,
QuotaUsageType,
SaveTableRequest,
SearchQueryOperators,
SearchFilterOperator,
SortOrder,
SortType,
StaticQuotaName,
@ -132,7 +132,7 @@ describe.each([
primaryDisplay: generator.word(),
query: [
{
operator: SearchQueryOperators.EQUAL,
operator: SearchFilterOperator.EQUAL,
field: "field",
value: "value",
},
@ -236,7 +236,7 @@ describe.each([
...view,
query: [
{
operator: SearchQueryOperators.EQUAL,
operator: SearchFilterOperator.EQUAL,
field: "newField",
value: "thatValue",
},
@ -263,7 +263,7 @@ describe.each([
primaryDisplay: generator.word(),
query: [
{
operator: SearchQueryOperators.EQUAL,
operator: SearchFilterOperator.EQUAL,
field: generator.word(),
value: generator.word(),
},
@ -341,7 +341,7 @@ describe.each([
tableId: generator.guid(),
query: [
{
operator: SearchQueryOperators.EQUAL,
operator: SearchFilterOperator.EQUAL,
field: "newField",
value: "thatValue",
},
@ -671,7 +671,7 @@ describe.each([
name: generator.guid(),
query: [
{
operator: SearchQueryOperators.EQUAL,
operator: SearchFilterOperator.EQUAL,
field: "two",
value: "bar2",
},

1
packages/server/src/automations/steps/triggerAutomationRun.ts
View File

@ -62,7 +62,6 @@ export const definition: AutomationStepSchema = {
}
export async function run({ inputs }: AutomationStepInput) {
// eslint-disable-next-line @typescript-eslint/no-unused-vars
const { automationId, ...fieldParams } = inputs.automation
if (await features.isTriggerAutomationRunEnabled()) {

2
packages/server/src/constants/index.ts
View File

@ -20,6 +20,7 @@ export enum FilterTypes {
NOT_EMPTY = "notEmpty",
CONTAINS = "contains",
NOT_CONTAINS = "notContains",
CONTAINS_ANY = "containsAny",
ONE_OF = "oneOf",
}
@ -30,6 +31,7 @@ export const NoEmptyFilterStrings = [
FilterTypes.NOT_EQUAL,
FilterTypes.CONTAINS,
FilterTypes.NOT_CONTAINS,
FilterTypes.CONTAINS_ANY,
]
export const CanSwitchTypes = [

1
packages/server/src/db/utils.ts
View File

@ -40,6 +40,7 @@ export const USER_METDATA_PREFIX = `${DocumentType.ROW}${SEPARATOR}${dbCore.Inte
export const LINK_USER_METADATA_PREFIX = `${DocumentType.LINK}${SEPARATOR}${dbCore.InternalTable.USER_METADATA}${SEPARATOR}`
export const TABLE_ROW_PREFIX = `${DocumentType.ROW}${SEPARATOR}${DocumentType.TABLE}`
export const AUTOMATION_LOG_PREFIX = `${DocumentType.AUTOMATION_LOG}${SEPARATOR}`
export const SQS_DATASOURCE_INTERNAL = "internal"
export const ViewName = dbCore.ViewName
export const InternalTables = dbCore.InternalTable
export const UNICODE_MAX = dbCore.UNICODE_MAX

2
packages/server/src/environment.ts
View File

@ -28,6 +28,7 @@ const DEFAULTS = {
PLUGINS_DIR: "/plugins",
FORKED_PROCESS_NAME: "main",
JS_RUNNER_MEMORY_LIMIT: 64,
COUCH_DB_SQL_URL: "http://localhost:4006",
}
const QUERY_THREAD_TIMEOUT =
@ -39,6 +40,7 @@ const environment = {
// important - prefer app port to generic port
PORT: process.env.APP_PORT || process.env.PORT,
COUCH_DB_URL: process.env.COUCH_DB_URL,
COUCH_DB_SQL_URL: process.env.COUCH_DB_SQL_URL || DEFAULTS.COUCH_DB_SQL_URL,
MINIO_URL: process.env.MINIO_URL,
WORKER_URL: process.env.WORKER_URL,
AWS_REGION: process.env.AWS_REGION,

3
packages/server/src/features.ts
View File

@ -1,11 +1,8 @@
import { features } from "@budibase/backend-core"
import env from "./environment"
// eslint-disable-next-line no-unused-vars
enum AppFeature {
// eslint-disable-next-line no-unused-vars
API = "api",
// eslint-disable-next-line no-unused-vars
AUTOMATIONS = "automations",
}

67
packages/server/src/integration-test/mysql.spec.ts
View File

@ -16,7 +16,6 @@ import {
getDatasource,
rawQuery,
} from "../integrations/tests/utils"
import { builderSocket } from "../websockets"
import { generator } from "@budibase/backend-core/tests"
// @ts-ignore
fetch.mockSearch()
@ -233,72 +232,6 @@ describe("mysql integrations", () => {
})
describe("POST /api/tables/", () => {
const emitDatasourceUpdateMock = jest.fn()
it("will emit the datasource entity schema with externalType to the front-end when adding a new column", async () => {
const addColumnToTable: TableRequest = {
type: "table",
sourceType: TableSourceType.EXTERNAL,
name: uniqueTableName(),
sourceId: datasource._id!,
primary: ["id"],
schema: {
id: {
type: FieldType.AUTO,
name: "id",
autocolumn: true,
},
new_column: {
type: FieldType.NUMBER,
name: "new_column",
},
},
_add: {
name: "new_column",
},
}
jest
.spyOn(builderSocket!, "emitDatasourceUpdate")
.mockImplementation(emitDatasourceUpdateMock)
await makeRequest("post", "/api/tables/", addColumnToTable)
const expectedTable: TableRequest = {
...addColumnToTable,
schema: {
id: {
type: FieldType.NUMBER,
name: "id",
autocolumn: true,
constraints: {
presence: false,
},
externalType: "int unsigned",
},
new_column: {
type: FieldType.NUMBER,
name: "new_column",
autocolumn: false,
constraints: {
presence: false,
},
externalType: "float(8,2)",
},
},
created: true,
_id: `${datasource._id}__${addColumnToTable.name}`,
}
delete expectedTable._add
expect(emitDatasourceUpdateMock).toHaveBeenCalledTimes(1)
const emittedDatasource: Datasource =
emitDatasourceUpdateMock.mock.calls[0][1]
expect(emittedDatasource.entities![expectedTable.name]).toEqual(
expectedTable
)
})
it("will rename a column", async () => {
await makeRequest("post", "/api/tables/", primaryMySqlTable)

12
packages/server/src/integrations/base/query.ts
View File

@ -2,6 +2,7 @@ import {
QueryJson,
Datasource,
DatasourcePlusQueryResponse,
RowOperations,
} from "@budibase/types"
import { getIntegration } from "../index"
import sdk from "../../sdk"
@ -10,6 +11,17 @@ export async function makeExternalQuery(
datasource: Datasource,
json: QueryJson
): Promise<DatasourcePlusQueryResponse> {
const entityId = json.endpoint.entityId,
tableName = json.meta.table.name,
tableId = json.meta.table._id
// case found during testing - make sure this doesn't happen again
if (
RowOperations.includes(json.endpoint.operation) &&
entityId !== tableId &&
entityId !== tableName
) {
throw new Error("Entity ID and table metadata do not align")
}
datasource = await sdk.datasources.enrich(datasource)
const Integration = await getIntegration(datasource.source)
// query is the opinionated function

91
packages/server/src/integrations/base/sql.ts
View File

@ -6,6 +6,7 @@ import {
SqlClient,
isValidFilter,
getNativeSql,
SqlStatements,
} from "../utils"
import SqlTableQueryBuilder from "./sqlTable"
import {
@ -22,6 +23,8 @@ import {
SortDirection,
SqlQueryBinding,
Table,
TableSourceType,
INTERNAL_TABLE_SOURCE_ID,
} from "@budibase/types"
import environment from "../../environment"
@ -135,6 +138,18 @@ function generateSelectStatement(
})
}
function getTableName(table?: Table): string | undefined {
// SQS uses the table ID rather than the table name
if (
table?.sourceType === TableSourceType.INTERNAL ||
table?.sourceId === INTERNAL_TABLE_SOURCE_ID
) {
return table?._id
} else {
return table?.name
}
}
class InternalBuilder {
private readonly client: string
@ -146,10 +161,20 @@ class InternalBuilder {
addFilters(
query: Knex.QueryBuilder,
filters: SearchFilters | undefined,
tableName: string,
table: Table,
opts: { aliases?: Record<string, string>; relationship?: boolean }
): Knex.QueryBuilder {
function getTableName(name: string) {
if (!filters) {
return query
}
filters = parseFilters(filters)
// if all or specified in filters, then everything is an or
const allOr = filters.allOr
const sqlStatements = new SqlStatements(this.client, table, { allOr })
const tableName =
this.client === SqlClient.SQL_LITE ? table._id! : table.name
function getTableAlias(name: string) {
const alias = opts.aliases?.[name]
return alias || name
}
@ -161,11 +186,11 @@ class InternalBuilder {
const updatedKey = dbCore.removeKeyNumbering(key)
const isRelationshipField = updatedKey.includes(".")
if (!opts.relationship && !isRelationshipField) {
fn(`${getTableName(tableName)}.${updatedKey}`, value)
fn(`${getTableAlias(tableName)}.${updatedKey}`, value)
}
if (opts.relationship && isRelationshipField) {
const [filterTableName, property] = updatedKey.split(".")
fn(`${getTableName(filterTableName)}.${property}`, value)
fn(`${getTableAlias(filterTableName)}.${property}`, value)
}
}
}
@ -233,18 +258,17 @@ class InternalBuilder {
(statement ? andOr : "") +
`LOWER(${likeKey(this.client, key)}) LIKE ?`
}
if (statement === "") {
return
}
// @ts-ignore
query = query[rawFnc](`${not}(${statement})`, value)
})
}
}
if (!filters) {
return query
}
filters = parseFilters(filters)
// if all or specified in filters, then everything is an or
const allOr = filters.allOr
if (filters.oneOf) {
iterate(filters.oneOf, (key, array) => {
const fnc = allOr ? "orWhereIn" : "whereIn"
@ -287,17 +311,11 @@ class InternalBuilder {
const lowValid = isValidFilter(value.low),
highValid = isValidFilter(value.high)
if (lowValid && highValid) {
// Use a between operator if we have 2 valid range values
const fnc = allOr ? "orWhereBetween" : "whereBetween"
query = query[fnc](key, [value.low, value.high])
query = sqlStatements.between(query, key, value.low, value.high)
} else if (lowValid) {
// Use just a single greater than operator if we only have a low
const fnc = allOr ? "orWhere" : "where"
query = query[fnc](key, ">", value.low)
query = sqlStatements.lte(query, key, value.low)
} else if (highValid) {
// Use just a single less than operator if we only have a high
const fnc = allOr ? "orWhere" : "where"
query = query[fnc](key, "<", value.high)
query = sqlStatements.gte(query, key, value.high)
}
})
}
@ -340,10 +358,11 @@ class InternalBuilder {
addSorting(query: Knex.QueryBuilder, json: QueryJson): Knex.QueryBuilder {
let { sort, paginate } = json
const table = json.meta?.table
const table = json.meta.table
const tableName = getTableName(table)
const aliases = json.tableAliases
const aliased =
table?.name && aliases?.[table.name] ? aliases[table.name] : table?.name
tableName && aliases?.[tableName] ? aliases[tableName] : table?.name
if (sort && Object.keys(sort || {}).length > 0) {
for (let [key, value] of Object.entries(sort)) {
const direction =
@ -453,14 +472,13 @@ class InternalBuilder {
): Knex.QueryBuilder {
const tableName = endpoint.entityId
const tableAlias = aliases?.[tableName]
let table: string | Record<string, string> = tableName
if (tableAlias) {
table = { [tableAlias]: tableName }
}
let query = knex(table)
if (endpoint.schema) {
query = query.withSchema(endpoint.schema)
}
const query = knex(
this.tableNameWithSchema(tableName, {
alias: tableAlias,
schema: endpoint.schema,
})
)
return query
}
@ -527,7 +545,7 @@ class InternalBuilder {
if (foundOffset) {
query = query.offset(foundOffset)
}
query = this.addFilters(query, filters, tableName, {
query = this.addFilters(query, filters, json.meta.table, {
aliases: tableAliases,
})
// add sorting to pre-query
@ -548,7 +566,7 @@ class InternalBuilder {
endpoint.schema,
tableAliases
)
return this.addFilters(query, filters, tableName, {
return this.addFilters(query, filters, json.meta.table, {
relationship: true,
aliases: tableAliases,
})
@ -558,7 +576,7 @@ class InternalBuilder {
const { endpoint, body, filters, tableAliases } = json
let query = this.knexWithAlias(knex, endpoint, tableAliases)
const parsedBody = parseBody(body)
query = this.addFilters(query, filters, endpoint.entityId, {
query = this.addFilters(query, filters, json.meta.table, {
aliases: tableAliases,
})
// mysql can't use returning
@ -572,7 +590,7 @@ class InternalBuilder {
delete(knex: Knex, json: QueryJson, opts: QueryOptions): Knex.QueryBuilder {
const { endpoint, filters, tableAliases } = json
let query = this.knexWithAlias(knex, endpoint, tableAliases)
query = this.addFilters(query, filters, endpoint.entityId, {
query = this.addFilters(query, filters, json.meta.table, {
aliases: tableAliases,
})
// mysql can't use returning
@ -664,7 +682,7 @@ class SqlQueryBuilder extends SqlTableQueryBuilder {
// when creating if an ID has been inserted need to make sure
// the id filter is enriched with it before trying to retrieve the row
checkLookupKeys(id: any, json: QueryJson) {
if (!id || !json.meta?.table || !json.meta.table.primary) {
if (!id || !json.meta.table || !json.meta.table.primary) {
return json
}
const primaryKey = json.meta.table.primary?.[0]
@ -724,12 +742,13 @@ class SqlQueryBuilder extends SqlTableQueryBuilder {
results: Record<string, any>[],
aliases?: Record<string, string>
): Record<string, any>[] {
const tableName = getTableName(table)
for (const [name, field] of Object.entries(table.schema)) {
if (!this._isJsonColumn(field)) {
continue
}
const tableName = aliases?.[table.name] || table.name
const fullName = `${tableName}.${name}`
const aliasedTableName = (tableName && aliases?.[tableName]) || tableName
const fullName = `${aliasedTableName}.${name}`
for (let row of results) {
if (typeof row[fullName] === "string") {
row[fullName] = JSON.parse(row[fullName])

23
packages/server/src/integrations/base/sqlTable.ts
View File

@ -1,19 +1,20 @@
import { Knex, knex } from "knex"
import {
RelationshipType,
FieldSubtype,
FieldType,
NumberFieldMetadata,
Operation,
QueryJson,
RelationshipType,
RenameColumn,
Table,
FieldType,
SqlQuery,
Table,
TableSourceType,
} from "@budibase/types"
import { breakExternalTableId, getNativeSql, SqlClient } from "../utils"
import { utils } from "@budibase/shared-core"
import SchemaBuilder = Knex.SchemaBuilder
import CreateTableBuilder = Knex.CreateTableBuilder
import { utils } from "@budibase/shared-core"
function isIgnoredType(type: FieldType) {
const ignored = [FieldType.LINK, FieldType.FORMULA]
@ -105,13 +106,13 @@ function generateSchema(
column.relationshipType !== RelationshipType.MANY_TO_MANY
) {
if (!column.foreignKey || !column.tableId) {
throw "Invalid relationship schema"
throw new Error("Invalid relationship schema")
}
const { tableName } = breakExternalTableId(column.tableId)
// @ts-ignore
const relatedTable = tables[tableName]
if (!relatedTable) {
throw "Referenced table doesn't exist"
throw new Error("Referenced table doesn't exist")
}
const relatedPrimary = relatedTable.primary[0]
const externalType = relatedTable.schema[relatedPrimary].externalType
@ -209,15 +210,19 @@ class SqlTableQueryBuilder {
let query: Knex.SchemaBuilder
if (!json.table || !json.meta || !json.meta.tables) {
throw "Cannot execute without table being specified"
throw new Error("Cannot execute without table being specified")
}
if (json.table.sourceType === TableSourceType.INTERNAL) {
throw new Error("Cannot perform table actions for SQS.")
}
switch (this._operation(json)) {
case Operation.CREATE_TABLE:
query = buildCreateTable(client, json.table, json.meta.tables)
break
case Operation.UPDATE_TABLE:
if (!json.meta || !json.meta.table) {
throw "Must specify old table for update"
throw new Error("Must specify old table for update")
}
// renameColumn does not work for MySQL, so return a raw query
if (this.sqlClient === SqlClient.MY_SQL && json.meta.renamed) {
@ -264,7 +269,7 @@ class SqlTableQueryBuilder {
query = buildDeleteTable(client, json.table)
break
default:
throw "Table operation is of unknown type"
throw new Error("Table operation is of unknown type")
}
return getNativeSql(query)
}

3
packages/server/src/integrations/googlesheets.ts
View File

@ -168,8 +168,7 @@ class GoogleSheetsIntegration implements DatasourcePlus {
return ""
}
// eslint-disable-next-line @typescript-eslint/no-unused-vars
getStringConcat(parts: string[]) {
getStringConcat(_parts: string[]) {
return ""
}

2
packages/server/src/integrations/oracle.ts
View File

@ -378,7 +378,7 @@ class OracleIntegration extends Sql implements DatasourcePlus {
try {
await connection.close()
} catch (err) {
console.error(err)
console.error("Error connecting to Oracle", err)
}
}
}

58
packages/server/src/integrations/tests/sql.spec.ts
View File

@ -1,14 +1,27 @@
import { SqlClient } from "../utils"
import Sql from "../base/sql"
import {
FieldType,
Operation,
QueryJson,
TableSourceType,
Table,
FieldType,
TableSourceType,
} from "@budibase/types"
const TABLE_NAME = "test"
const TABLE: Table = {
type: "table",
sourceType: TableSourceType.EXTERNAL,
sourceId: "SOURCE_ID",
schema: {
id: {
name: "id",
type: FieldType.NUMBER,
},
},
name: TABLE_NAME,
primary: ["id"],
}
function endpoint(table: any, operation: any) {
return {
@ -25,6 +38,10 @@ function generateReadJson({
sort,
paginate,
}: any = {}): QueryJson {
const tableObj = { ...TABLE }
if (table) {
tableObj.name = table
}
return {
endpoint: endpoint(table || TABLE_NAME, "READ"),
resource: {
@ -34,14 +51,7 @@ function generateReadJson({
sort: sort || {},
paginate: paginate || {},
meta: {
table: {
type: "table",
sourceType: TableSourceType.EXTERNAL,
sourceId: "SOURCE_ID",
schema: {},
name: table || TABLE_NAME,
primary: ["id"],
} as any,
table: tableObj,
},
}
}
@ -49,6 +59,9 @@ function generateReadJson({
function generateCreateJson(table = TABLE_NAME, body = {}): QueryJson {
return {
endpoint: endpoint(table, "CREATE"),
meta: {
table: TABLE,
},
body,
}
}
@ -58,7 +71,15 @@ function generateUpdateJson({
body = {},
filters = {},
meta = {},
}: {
table: string
body?: any
filters?: any
meta?: any
}): QueryJson {
if (!meta.table) {
meta.table = TABLE
}
return {
endpoint: endpoint(table, "UPDATE"),
filters,
@ -70,6 +91,9 @@ function generateUpdateJson({
function generateDeleteJson(table = TABLE_NAME, filters = {}): QueryJson {
return {
endpoint: endpoint(table, "DELETE"),
meta: {
table: TABLE,
},
filters,
}
}
@ -102,6 +126,9 @@ function generateRelationshipJson(config: { schema?: string } = {}): QueryJson {
},
],
extra: { idFilter: {} },
meta: {
table: TABLE,
},
}
}
@ -136,6 +163,9 @@ function generateManyRelationshipJson(config: { schema?: string } = {}) {
},
],
extra: { idFilter: {} },
meta: {
table: TABLE,
},
}
}
@ -319,7 +349,7 @@ describe("SQL query builder", () => {
)
expect(query).toEqual({
bindings: [date, limit],
sql: `select * from (select * from "${TABLE_NAME}" where "${TABLE_NAME}"."property" > $1 limit $2) as "${TABLE_NAME}"`,
sql: `select * from (select * from "${TABLE_NAME}" where "${TABLE_NAME}"."property" >= $1 limit $2) as "${TABLE_NAME}"`,
})
})
@ -338,7 +368,7 @@ describe("SQL query builder", () => {
)
expect(query).toEqual({
bindings: [date, limit],
sql: `select * from (select * from "${TABLE_NAME}" where "${TABLE_NAME}"."property" < $1 limit $2) as "${TABLE_NAME}"`,
sql: `select * from (select * from "${TABLE_NAME}" where "${TABLE_NAME}"."property" <= $1 limit $2) as "${TABLE_NAME}"`,
})
})
@ -572,7 +602,7 @@ describe("SQL query builder", () => {
)
expect(query).toEqual({
bindings: ["2000-01-01 00:00:00", 500],
sql: `select * from (select * from "${TABLE_NAME}" where "${TABLE_NAME}"."dob" > $1 limit $2) as "${TABLE_NAME}"`,
sql: `select * from (select * from "${TABLE_NAME}" where "${TABLE_NAME}"."dob" >= $1 limit $2) as "${TABLE_NAME}"`,
})
})
@ -591,7 +621,7 @@ describe("SQL query builder", () => {
)
expect(query).toEqual({
bindings: ["2010-01-01 00:00:00", 500],
sql: `select * from (select * from "${TABLE_NAME}" where "${TABLE_NAME}"."dob" < $1 limit $2) as "${TABLE_NAME}"`,
sql: `select * from (select * from "${TABLE_NAME}" where "${TABLE_NAME}"."dob" <= $1 limit $2) as "${TABLE_NAME}"`,
})
})

22
packages/server/src/integrations/tests/sqlAlias.spec.ts
View File

@ -4,12 +4,26 @@ import {
QueryJson,
SourceName,
SqlQuery,
Table,
TableSourceType,
} from "@budibase/types"
import { join } from "path"
import Sql from "../base/sql"
import { SqlClient } from "../utils"
import AliasTables from "../../api/controllers/row/alias"
import { generator } from "@budibase/backend-core/tests"
import sdk from "../../sdk"
// this doesn't exist strictly
const TABLE: Table = {
type: "table",
sourceType: TableSourceType.EXTERNAL,
sourceId: "SOURCE_ID",
schema: {},
name: "tableName",
primary: ["id"],
}
const AliasTables = sdk.rows.AliasTables
function multiline(sql: string) {
return sql.replace(/\n/g, "").replace(/ +/g, " ")
@ -103,7 +117,8 @@ describe("Captures of real examples", () => {
let query = new Sql(SqlClient.POSTGRES, limit)._query(queryJson)
const filters = queryJson.filters
const notEqualsValue = Object.values(filters?.notEqual!)[0]
const rangeValue = Object.values(filters?.range!)[0]
const rangeValue: { high?: string | number; low?: string | number } =
Object.values(filters?.range!)[0]
const equalValue = Object.values(filters?.equal!)[0]
expect(query).toEqual({
@ -220,6 +235,9 @@ describe("Captures of real examples", () => {
resource: {
fields,
},
meta: {
table: TABLE,
},
}
}

2
packages/server/src/integrations/utils/index.ts Normal file
View File

@ -0,0 +1,2 @@
export * from "./utils"
export { SqlStatements } from "./sqlStatements"

80
packages/server/src/integrations/utils/sqlStatements.ts Normal file
View File

@ -0,0 +1,80 @@
import { FieldType, Table, FieldSchema } from "@budibase/types"
import { SqlClient } from "./utils"
import { Knex } from "knex"
export class SqlStatements {
client: string
table: Table
allOr: boolean | undefined
constructor(
client: string,
table: Table,
{ allOr }: { allOr?: boolean } = {}
) {
this.client = client
this.table = table
this.allOr = allOr
}
getField(key: string): FieldSchema | undefined {
const fieldName = key.split(".")[1]
return this.table.schema[fieldName]
}
between(
query: Knex.QueryBuilder,
key: string,
low: number | string,
high: number | string
) {
// Use a between operator if we have 2 valid range values
const field = this.getField(key)
if (
field?.type === FieldType.BIGINT &&
this.client === SqlClient.SQL_LITE
) {
query = query.whereRaw(
`CAST(${key} AS INTEGER) BETWEEN CAST(? AS INTEGER) AND CAST(? AS INTEGER)`,
[low, high]
)
} else {
const fnc = this.allOr ? "orWhereBetween" : "whereBetween"
query = query[fnc](key, [low, high])
}
return query
}
lte(query: Knex.QueryBuilder, key: string, low: number | string) {
// Use just a single greater than operator if we only have a low
const field = this.getField(key)
if (
field?.type === FieldType.BIGINT &&
this.client === SqlClient.SQL_LITE
) {
query = query.whereRaw(`CAST(${key} AS INTEGER) >= CAST(? AS INTEGER)`, [
low,
])
} else {
const fnc = this.allOr ? "orWhere" : "where"
query = query[fnc](key, ">=", low)
}
return query
}
gte(query: Knex.QueryBuilder, key: string, high: number | string) {
const field = this.getField(key)
// Use just a single less than operator if we only have a high
if (
field?.type === FieldType.BIGINT &&
this.client === SqlClient.SQL_LITE
) {
query = query.whereRaw(`CAST(${key} AS INTEGER) <= CAST(? AS INTEGER)`, [
high,
])
} else {
const fnc = this.allOr ? "orWhere" : "where"
query = query[fnc](key, "<=", high)
}
return query
}
}

175
packages/server/src/integrations/utils.ts → packages/server/src/integrations/utils/utils.ts
View File

@ -4,18 +4,40 @@ import {
Datasource,
FieldType,
TableSourceType,
FieldSchema,
} from "@budibase/types"
import { DocumentType, SEPARATOR } from "../db/utils"
import { InvalidColumns, DEFAULT_BB_DATASOURCE_ID } from "../constants"
import { helpers } from "@budibase/shared-core"
import env from "../environment"
import { DocumentType, SEPARATOR } from "../../db/utils"
import { InvalidColumns, DEFAULT_BB_DATASOURCE_ID } from "../../constants"
import { helpers, utils } from "@budibase/shared-core"
import env from "../../environment"
import { Knex } from "knex"
const DOUBLE_SEPARATOR = `${SEPARATOR}${SEPARATOR}`
const ROW_ID_REGEX = /^\[.*]$/g
const ENCODED_SPACE = encodeURIComponent(" ")
const SQL_NUMBER_TYPE_MAP = {
type PrimitiveTypes =
| FieldType.STRING
| FieldType.NUMBER
| FieldType.BOOLEAN
| FieldType.DATETIME
| FieldType.JSON
| FieldType.BIGINT
| FieldType.OPTIONS
function isPrimitiveType(type: FieldType): type is PrimitiveTypes {
return [
FieldType.STRING,
FieldType.NUMBER,
FieldType.BOOLEAN,
FieldType.DATETIME,
FieldType.JSON,
FieldType.BIGINT,
FieldType.OPTIONS,
].includes(type)
}
const SQL_NUMBER_TYPE_MAP: Record<string, PrimitiveTypes> = {
integer: FieldType.NUMBER,
int: FieldType.NUMBER,
decimal: FieldType.NUMBER,
@ -35,7 +57,7 @@ const SQL_NUMBER_TYPE_MAP = {
smallmoney: FieldType.NUMBER,
}
const SQL_DATE_TYPE_MAP = {
const SQL_DATE_TYPE_MAP: Record<string, PrimitiveTypes> = {
timestamp: FieldType.DATETIME,
time: FieldType.DATETIME,
datetime: FieldType.DATETIME,
@ -46,7 +68,7 @@ const SQL_DATE_TYPE_MAP = {
const SQL_DATE_ONLY_TYPES = ["date"]
const SQL_TIME_ONLY_TYPES = ["time"]
const SQL_STRING_TYPE_MAP = {
const SQL_STRING_TYPE_MAP: Record<string, PrimitiveTypes> = {
varchar: FieldType.STRING,
char: FieldType.STRING,
nchar: FieldType.STRING,
@ -58,22 +80,22 @@ const SQL_STRING_TYPE_MAP = {
text: FieldType.STRING,
}
const SQL_BOOLEAN_TYPE_MAP = {
const SQL_BOOLEAN_TYPE_MAP: Record<string, PrimitiveTypes> = {
boolean: FieldType.BOOLEAN,
bit: FieldType.BOOLEAN,
tinyint: FieldType.BOOLEAN,
}
const SQL_OPTIONS_TYPE_MAP = {
const SQL_OPTIONS_TYPE_MAP: Record<string, PrimitiveTypes> = {
"user-defined": FieldType.OPTIONS,
}
const SQL_MISC_TYPE_MAP = {
const SQL_MISC_TYPE_MAP: Record<string, PrimitiveTypes> = {
json: FieldType.JSON,
bigint: FieldType.BIGINT,
}
const SQL_TYPE_MAP = {
const SQL_TYPE_MAP: Record<string, PrimitiveTypes> = {
...SQL_NUMBER_TYPE_MAP,
...SQL_DATE_TYPE_MAP,
...SQL_STRING_TYPE_MAP,
@ -239,14 +261,14 @@ export function generateColumnDefinition(config: {
constraints.inclusion = options
}
const schema: any = {
const schema: FieldSchema = {
type: foundType,
externalType,
autocolumn,
name,
constraints,
}
if (foundType === FieldType.DATETIME) {
if (schema.type === FieldType.DATETIME) {
schema.dateOnly = SQL_DATE_ONLY_TYPES.includes(lowerCaseType)
schema.timeOnly = SQL_TIME_ONLY_TYPES.includes(lowerCaseType)
}
@ -274,59 +296,6 @@ export function isIsoDateString(str: string) {
return d.toISOString() === trimmedValue
}
/**
* This function will determine whether a column is a relationship and whether it
* is currently valid. The reason for the validity check is that tables can be deleted
* outside of Budibase control and if this is the case it will break Budibase relationships.
* The tableIds is a list passed down from the main finalise tables function, which is
* based on the tables that have just been fetched. This will only really be used on subsequent
* fetches to the first one - if the user is periodically refreshing Budibase knowledge of tables.
* @param column The column to check, to see if it is a valid relationship.
* @param tableIds The IDs of the tables which currently exist.
*/
export function shouldCopyRelationship(
column: { type: string; tableId?: string },
tableIds: string[]
) {
return (
column.type === FieldType.LINK &&
column.tableId &&
tableIds.includes(column.tableId)
)
}
/**
* Similar function to the shouldCopyRelationship function, but instead this looks for options and boolean
* types. It is possible to switch a string -> options and a number -> boolean (and vice versus) need to make
* sure that these get copied over when tables are fetched. Also checks whether they are still valid, if a
* column has changed type in the external database then copying it over may not be possible.
* @param column The column to check for options or boolean type.
* @param fetchedColumn The fetched column to check for the type in the external database.
*/
export function shouldCopySpecialColumn(
column: { type: string },
fetchedColumn: { type: string } | undefined
) {
const isFormula = column.type === FieldType.FORMULA
const specialTypes = [
FieldType.OPTIONS,
FieldType.LONGFORM,
FieldType.ARRAY,
FieldType.FORMULA,
FieldType.BB_REFERENCE,
]
// column has been deleted, remove - formulas will never exist, always copy
if (!isFormula && column && !fetchedColumn) {
return false
}
const fetchedIsNumber =
!fetchedColumn || fetchedColumn.type === FieldType.NUMBER
return (
specialTypes.indexOf(column.type as FieldType) !== -1 ||
(fetchedIsNumber && column.type === FieldType.BOOLEAN)
)
}
/**
* Looks for columns which need to be copied over into the new table definitions, like relationships,
* options types and views.
@ -348,6 +317,9 @@ function copyExistingPropsOver(
if (entities[tableName]?.created) {
table.created = entities[tableName]?.created
}
if (entities[tableName]?.constrained) {
table.constrained = entities[tableName]?.constrained
}
table.views = entities[tableName].views
@ -356,12 +328,73 @@ function copyExistingPropsOver(
if (!Object.prototype.hasOwnProperty.call(existingTableSchema, key)) {
continue
}
const column = existingTableSchema[key]
if (
shouldCopyRelationship(column, tableIds) ||
shouldCopySpecialColumn(column, table.schema[key])
) {
table.schema[key] = existingTableSchema[key]
const existingColumnType = column?.type
const updatedColumnType = table.schema[key]?.type
const keepIfType = (...validTypes: PrimitiveTypes[]) => {
return (
isPrimitiveType(updatedColumnType) &&
table.schema[key] &&
validTypes.includes(updatedColumnType)
)
}
let shouldKeepSchema = false
switch (existingColumnType) {
case FieldType.FORMULA:
case FieldType.AUTO:
case FieldType.INTERNAL:
shouldKeepSchema = true
break
case FieldType.LINK:
shouldKeepSchema =
existingColumnType === FieldType.LINK &&
tableIds.includes(column.tableId)
break
case FieldType.STRING:
case FieldType.OPTIONS:
case FieldType.LONGFORM:
case FieldType.BARCODEQR:
shouldKeepSchema = keepIfType(FieldType.STRING)
break
case FieldType.NUMBER:
case FieldType.BOOLEAN:
shouldKeepSchema = keepIfType(FieldType.BOOLEAN, FieldType.NUMBER)
break
case FieldType.ARRAY:
case FieldType.ATTACHMENTS:
case FieldType.ATTACHMENT_SINGLE:
case FieldType.JSON:
case FieldType.BB_REFERENCE:
shouldKeepSchema = keepIfType(FieldType.JSON, FieldType.STRING)
break
case FieldType.DATETIME:
shouldKeepSchema = keepIfType(FieldType.DATETIME, FieldType.STRING)
break
case FieldType.BIGINT:
shouldKeepSchema = keepIfType(FieldType.BIGINT, FieldType.NUMBER)
break
default:
utils.unreachable(existingColumnType)
}
if (shouldKeepSchema) {
table.schema[key] = {
...existingTableSchema[key],
externalType:
existingTableSchema[key].externalType ||
table.schema[key]?.externalType,
}
}
}
}

1
packages/server/src/jsRunner/index.ts
View File

@ -35,7 +35,6 @@ export function init() {
// Because we can't pass functions into an Isolate, we remove them from
// the passed context and rely on the withHelpers() method to add them
// back in.
// eslint-disable-next-line @typescript-eslint/no-unused-vars
const { helpers, snippets, ...rest } = ctx
return vm.withContext(rest, () => vm.execute(js))
} catch (error: any) {

3
packages/server/src/sdk/app/datasources/datasources.ts
View File

@ -348,8 +348,7 @@ const preSaveAction: Partial<Record<SourceName, any>> = {
* Make sure all datasource entities have a display name selected
*/
export function setDefaultDisplayColumns(datasource: Datasource) {
//
for (let entity of Object.values(datasource.entities || {})) {
for (const entity of Object.values(datasource.entities || {})) {
if (entity.primaryDisplay) {
continue
}

2
packages/server/src/sdk/app/rows/index.ts
View File

@ -3,6 +3,7 @@ import * as rows from "./rows"
import * as search from "./search"
import * as utils from "./utils"
import * as external from "./external"
import AliasTables from "./sqlAlias"
export default {
...attachments,
@ -10,4 +11,5 @@ export default {
...search,
utils,
external,
AliasTables,
}

17
packages/server/src/sdk/app/rows/search.ts
View File

@ -13,6 +13,8 @@ import * as sqs from "./search/sqs"
import env from "../../../environment"
import { ExportRowsParams, ExportRowsResult } from "./search/types"
import { dataFilters } from "@budibase/shared-core"
import sdk from "../../index"
import { searchInputMapping } from "./search/utils"
export { isValidFilter } from "../../../integrations/utils"
@ -29,6 +31,10 @@ function pickApi(tableId: any) {
return internal
}
function isEmptyArray(value: any) {
return Array.isArray(value) && value.length === 0
}
// don't do a pure falsy check, as 0 is included
// https://github.com/Budibase/budibase/issues/10118
export function removeEmptyFilters(filters: SearchFilters) {
@ -47,7 +53,7 @@ export function removeEmptyFilters(filters: SearchFilters) {
for (let [key, value] of Object.entries(
filters[filterType] as object
)) {
if (value == null || value === "") {
if (value == null || value === "" || isEmptyArray(value)) {
// @ts-ignore
delete filters[filterField][key]
}
@ -72,12 +78,15 @@ export async function search(
}
}
const table = await sdk.tables.getTable(options.tableId)
options = searchInputMapping(table, options)
if (isExternalTable) {
return external.search(options)
return external.search(options, table)
} else if (env.SQS_SEARCH_ENABLE) {
return sqs.search(options)
return sqs.search(options, table)
} else {
return internal.search(options)
return internal.search(options, table)
}
}

13
packages/server/src/sdk/app/rows/search/external.ts
View File

@ -8,6 +8,7 @@ import {
SearchFilters,
RowSearchParams,
SearchResponse,
Table,
} from "@budibase/types"
import * as exporters from "../../../../api/controllers/view/exporters"
import { handleRequest } from "../../../../api/controllers/row/external"
@ -18,13 +19,13 @@ import {
import { utils } from "@budibase/shared-core"
import { ExportRowsParams, ExportRowsResult } from "./types"
import { HTTPError, db } from "@budibase/backend-core"
import { searchInputMapping } from "./utils"
import pick from "lodash/pick"
import { outputProcessing } from "../../../../utilities/rowProcessor"
import sdk from "../../../"
export async function search(
options: RowSearchParams
options: RowSearchParams,
table: Table
): Promise<SearchResponse<Row>> {
const { tableId } = options
const { paginate, query, ...params } = options
@ -68,8 +69,6 @@ export async function search(
}
try {
const table = await sdk.tables.getTable(tableId)
options = searchInputMapping(table, options)
let rows = await handleRequest(Operation.READ, tableId, {
filters: query,
sort,
@ -150,11 +149,15 @@ export async function exportRows(
}
const datasource = await sdk.datasources.get(datasourceId!)
const table = await sdk.tables.getTable(tableId)
if (!datasource || !datasource.entities) {
throw new HTTPError("Datasource has not been configured for plus API.", 400)
}
let result = await search({ tableId, query: requestQuery, sort, sortOrder })
let result = await search(
{ tableId, query: requestQuery, sort, sortOrder },
table
)
let rows: Row[] = []
let headers

22
packages/server/src/sdk/app/rows/search/internal.ts
View File

@ -1,6 +1,6 @@
import { context, db, HTTPError } from "@budibase/backend-core"
import env from "../../../../environment"
import { fullSearch, paginatedSearch, searchInputMapping } from "./utils"
import { fullSearch, paginatedSearch } from "./utils"
import { getRowParams, InternalTables } from "../../../../db/utils"
import {
Database,
@ -33,7 +33,8 @@ import pick from "lodash/pick"
import { breakRowIdField } from "../../../../integrations/utils"
export async function search(
options: RowSearchParams
options: RowSearchParams,
table: Table
): Promise<SearchResponse<Row>> {
const { tableId } = options
@ -51,8 +52,6 @@ export async function search(
query: {},
}
let table = await sdk.tables.getTable(tableId)
options = searchInputMapping(table, options)
if (params.sort && !params.sortType) {
const schema = table.schema
const sortField = schema[params.sort]
@ -122,12 +121,15 @@ export async function exportRows(
result = await outputProcessing<Row[]>(table, response)
} else if (query) {
let searchResponse = await search({
tableId,
query,
sort,
sortOrder,
})
let searchResponse = await search(
{
tableId,
query,
sort,
sortOrder,
},
table
)
result = searchResponse.rows
}

101
packages/server/src/sdk/app/rows/search/sqs.ts
View File

@ -20,7 +20,12 @@ import {
} from "../../../../api/controllers/row/utils"
import sdk from "../../../index"
import { context } from "@budibase/backend-core"
import { CONSTANT_INTERNAL_ROW_COLS } from "../../../../db/utils"
import {
CONSTANT_INTERNAL_ROW_COLS,
SQS_DATASOURCE_INTERNAL,
} from "../../../../db/utils"
import AliasTables from "../sqlAlias"
import { outputProcessing } from "../../../../utilities/rowProcessor"
function buildInternalFieldList(
table: Table,
@ -31,19 +36,19 @@ function buildInternalFieldList(
fieldList = fieldList.concat(
CONSTANT_INTERNAL_ROW_COLS.map(col => `${table._id}.${col}`)
)
if (opts.relationships) {
for (let col of Object.values(table.schema)) {
if (col.type === FieldType.LINK) {
const linkCol = col as RelationshipFieldMetadata
const relatedTable = tables.find(
table => table._id === linkCol.tableId
)!
fieldList = fieldList.concat(
buildInternalFieldList(relatedTable, tables, { relationships: false })
)
} else {
fieldList.push(`${table._id}.${col.name}`)
}
for (let col of Object.values(table.schema)) {
const isRelationship = col.type === FieldType.LINK
if (!opts.relationships && isRelationship) {
continue
}
if (isRelationship) {
const linkCol = col as RelationshipFieldMetadata
const relatedTable = tables.find(table => table._id === linkCol.tableId)!
fieldList = fieldList.concat(
buildInternalFieldList(relatedTable, tables, { relationships: false })
)
} else {
fieldList.push(`${table._id}.${col.name}`)
}
}
return fieldList
@ -94,14 +99,14 @@ function buildTableMap(tables: Table[]) {
}
export async function search(
options: RowSearchParams
options: RowSearchParams,
table: Table
): Promise<SearchResponse<Row>> {
const { tableId, paginate, query, ...params } = options
const { paginate, query, ...params } = options
const builder = new SqlQueryBuilder(SqlClient.SQL_LITE)
const allTables = await sdk.tables.getAllInternalTables()
const allTablesMap = buildTableMap(allTables)
const table = allTables.find(table => table._id === tableId)
if (!table) {
throw new Error("Unable to find table")
}
@ -111,7 +116,7 @@ export async function search(
const request: QueryJson = {
endpoint: {
// not important, we query ourselves
datasourceId: "internal",
datasourceId: SQS_DATASOURCE_INTERNAL,
entityId: table._id!,
operation: Operation.READ,
},
@ -154,37 +159,47 @@ export async function search(
}
}
try {
const query = builder._query(request, {
disableReturning: true,
const alias = new AliasTables(allTables.map(table => table.name))
const rows = await alias.queryWithAliasing(request, async json => {
const query = builder._query(json, {
disableReturning: true,
})
if (Array.isArray(query)) {
throw new Error("SQS cannot currently handle multiple queries")
}
let sql = query.sql,
bindings = query.bindings
// quick hack for docIds
sql = sql.replace(/`doc1`.`rowId`/g, "`doc1.rowId`")
sql = sql.replace(/`doc2`.`rowId`/g, "`doc2.rowId`")
const db = context.getAppDB()
return await db.sql<Row>(sql, bindings)
})
if (Array.isArray(query)) {
throw new Error("SQS cannot currently handle multiple queries")
}
let sql = query.sql,
bindings = query.bindings
// quick hack for docIds
sql = sql.replace(/`doc1`.`rowId`/g, "`doc1.rowId`")
sql = sql.replace(/`doc2`.`rowId`/g, "`doc2.rowId`")
const db = context.getAppDB()
const rows = await db.sql<Row>(sql, bindings)
// process from the format of tableId.column to expected format
const processed = await sqlOutputProcessing(
rows,
table!,
allTablesMap,
relationships,
{
sqs: true,
}
)
return {
rows: await sqlOutputProcessing(
rows,
table!,
allTablesMap,
relationships,
{
sqs: true,
}
),
// final row processing for response
rows: await outputProcessing<Row[]>(table, processed, {
preserveLinks: true,
squash: true,
}),
}
} catch (err: any) {
const msg = typeof err === "string" ? err : err.message
throw new Error(`Unable to search by SQL - ${msg}`)
throw new Error(`Unable to search by SQL - ${msg}`, { cause: err })
}
}

6
packages/server/src/sdk/app/rows/search/tests/external.spec.ts
View File

@ -112,7 +112,7 @@ describe("external search", () => {
tableId,
query: {},
}
const result = await search(searchParams)
const result = await search(searchParams, config.table!)
expect(result.rows).toHaveLength(10)
expect(result.rows).toEqual(
@ -130,7 +130,7 @@ describe("external search", () => {
query: {},
fields: ["name", "age"],
}
const result = await search(searchParams)
const result = await search(searchParams, config.table!)
expect(result.rows).toHaveLength(10)
expect(result.rows).toEqual(
@ -157,7 +157,7 @@ describe("external search", () => {
},
},
}
const result = await search(searchParams)
const result = await search(searchParams, config.table!)
expect(result.rows).toHaveLength(3)
expect(result.rows.map(row => row.id)).toEqual([1, 4, 8])

4
packages/server/src/sdk/app/rows/search/tests/internal.spec.ts
View File

@ -81,7 +81,7 @@ describe("internal", () => {
tableId,
query: {},
}
const result = await search(searchParams)
const result = await search(searchParams, config.table!)
expect(result.rows).toHaveLength(10)
expect(result.rows).toEqual(
@ -99,7 +99,7 @@ describe("internal", () => {
query: {},
fields: ["name", "age"],
}
const result = await search(searchParams)
const result = await search(searchParams, config.table!)
expect(result.rows).toHaveLength(10)
expect(result.rows).toEqual(

4
packages/server/src/sdk/app/rows/search/tests/lucene.ts → packages/server/src/sdk/app/rows/search/tests/lucene.spec.ts
View File

@ -160,7 +160,7 @@ describe("internal search", () => {
const response = await search.paginatedSearch(
{
contains: {
column: "a",
column: ["a"],
colArr: [1, 2, 3],
},
},
@ -168,7 +168,7 @@ describe("internal search", () => {
)
checkLucene(
response,
`(*:* AND column:a AND colArr:(1 AND 2 AND 3))`,
`(*:* AND column:(a) AND colArr:(1 AND 2 AND 3))`,
PARAMS
)
})

28
packages/server/src/api/controllers/row/alias.ts → packages/server/src/sdk/app/rows/sqlAlias.ts
View File

@ -6,11 +6,12 @@ import {
Row,
SearchFilters,
} from "@budibase/types"
import { getSQLClient } from "../../../sdk/app/rows/utils"
import { getSQLClient } from "./utils"
import { cloneDeep } from "lodash"
import sdk from "../../../sdk"
import datasources from "../datasources"
import { makeExternalQuery } from "../../../integrations/base/query"
import { SqlClient } from "../../../integrations/utils"
import { SQS_DATASOURCE_INTERNAL } from "../../../db/utils"
const WRITE_OPERATIONS: Operation[] = [
Operation.CREATE,
@ -156,12 +157,19 @@ export default class AliasTables {
}
async queryWithAliasing(
json: QueryJson
json: QueryJson,
queryFn?: (json: QueryJson) => Promise<DatasourcePlusQueryResponse>
): Promise<DatasourcePlusQueryResponse> {
const datasourceId = json.endpoint.datasourceId
const datasource = await sdk.datasources.get(datasourceId)
const isSqs = datasourceId === SQS_DATASOURCE_INTERNAL
let aliasingEnabled: boolean, datasource: Datasource | undefined
if (isSqs) {
aliasingEnabled = true
} else {
datasource = await datasources.get(datasourceId)
aliasingEnabled = this.isAliasingEnabled(json, datasource)
}
const aliasingEnabled = this.isAliasingEnabled(json, datasource)
if (aliasingEnabled) {
json = cloneDeep(json)
// run through the query json to update anywhere a table may be used
@ -207,7 +215,15 @@ export default class AliasTables {
}
json.tableAliases = invertedTableAliases
}
const response = await makeExternalQuery(datasource, json)
let response: DatasourcePlusQueryResponse
if (datasource && !isSqs) {
response = await makeExternalQuery(datasource, json)
} else if (queryFn) {
response = await queryFn(json)
} else {
throw new Error("No supplied method to perform aliased query")
}
if (Array.isArray(response) && aliasingEnabled) {
return this.reverse(response)
} else {

6
packages/server/src/sdk/app/rows/utils.ts
View File

@ -52,6 +52,12 @@ export async function getDatasourceAndQuery(
): Promise<DatasourcePlusQueryResponse> {
const datasourceId = json.endpoint.datasourceId
const datasource = await sdk.datasources.get(datasourceId)
const table = datasource.entities?.[json.endpoint.entityId]
if (!json.meta && table) {
json.meta = {
table,
}
}
return makeExternalQuery(datasource, json)
}

18
packages/server/src/sdk/app/tables/external/index.ts vendored
View File

@ -3,7 +3,6 @@ import {
Operation,
RelationshipType,
RenameColumn,
AddColumn,
Table,
TableRequest,
ViewV2,
@ -33,7 +32,7 @@ import * as viewSdk from "../../views"
export async function save(
datasourceId: string,
update: Table,
opts?: { tableId?: string; renaming?: RenameColumn; adding?: AddColumn }
opts?: { tableId?: string; renaming?: RenameColumn }
) {
let tableToSave: TableRequest = {
...update,
@ -52,6 +51,12 @@ export async function save(
!oldTable &&
(tableToSave.primary == null || tableToSave.primary.length === 0)
) {
if (tableToSave.schema.id) {
throw new Error(
"External tables with no `primary` column set will define an `id` column, but we found an `id` column in the supplied schema. Either set a `primary` column or remove the `id` column."
)
}
tableToSave.primary = ["id"]
tableToSave.schema.id = {
type: FieldType.NUMBER,
@ -179,14 +184,7 @@ export async function save(
// remove the rename prop
delete tableToSave._rename
// if adding a new column, we need to rebuild the schema for that table to get the 'externalType' of the column
if (opts?.adding) {
datasource.entities[tableToSave.name] = (
await datasourceSdk.buildFilteredSchema(datasource, [tableToSave.name])
).tables[tableToSave.name]
} else {
datasource.entities[tableToSave.name] = tableToSave
}
datasource.entities[tableToSave.name] = tableToSave
// store it into couch now for budibase reference
await db.put(populateExternalTableSchemas(datasource))

66
packages/server/src/sdk/app/tables/internal/sqs.ts
View File

@ -1,8 +1,19 @@
import { context, SQLITE_DESIGN_DOC_ID } from "@budibase/backend-core"
import { FieldType, SQLiteDefinition, SQLiteType, Table } from "@budibase/types"
import {
FieldType,
RelationshipFieldMetadata,
SQLiteDefinition,
SQLiteTable,
SQLiteTables,
SQLiteType,
Table,
} from "@budibase/types"
import { cloneDeep } from "lodash"
import tablesSdk from "../"
import { CONSTANT_INTERNAL_ROW_COLS } from "../../../../db/utils"
import {
CONSTANT_INTERNAL_ROW_COLS,
generateJunctionTableID,
} from "../../../../db/utils"
const BASIC_SQLITE_DOC: SQLiteDefinition = {
_id: SQLITE_DESIGN_DOC_ID,
@ -31,14 +42,47 @@ const FieldTypeMap: Record<FieldType, SQLiteType> = {
[FieldType.ATTACHMENT_SINGLE]: SQLiteType.BLOB,
[FieldType.ARRAY]: SQLiteType.BLOB,
[FieldType.LINK]: SQLiteType.BLOB,
[FieldType.BIGINT]: SQLiteType.REAL,
[FieldType.BIGINT]: SQLiteType.TEXT,
// TODO: consider the difference between multi-user and single user types (subtyping)
[FieldType.BB_REFERENCE]: SQLiteType.TEXT,
}
function mapTable(table: Table): { [key: string]: SQLiteType } {
function buildRelationshipDefinitions(
table: Table,
relationshipColumn: RelationshipFieldMetadata
): {
tableId: string
definition: SQLiteTable
} {
const tableId = table._id!,
relatedTableId = relationshipColumn.tableId
return {
tableId: generateJunctionTableID(tableId, relatedTableId),
definition: {
["doc1.rowId"]: SQLiteType.TEXT,
["doc1.tableId"]: SQLiteType.TEXT,
["doc1.fieldName"]: SQLiteType.TEXT,
["doc2.rowId"]: SQLiteType.TEXT,
["doc2.tableId"]: SQLiteType.TEXT,
["doc2.fieldName"]: SQLiteType.TEXT,
tableId: SQLiteType.TEXT,
},
}
}
// this can generate relationship tables as part of the mapping
function mapTable(table: Table): SQLiteTables {
const tables: SQLiteTables = {}
const fields: Record<string, SQLiteType> = {}
for (let [key, column] of Object.entries(table.schema)) {
// relationships should be handled differently
if (column.type === FieldType.LINK) {
const { tableId, definition } = buildRelationshipDefinitions(
table,
column
)
tables[tableId] = { fields: definition }
}
if (!FieldTypeMap[column.type]) {
throw new Error(`Unable to map type "${column.type}" to SQLite type`)
}
@ -49,10 +93,12 @@ function mapTable(table: Table): { [key: string]: SQLiteType } {
CONSTANT_INTERNAL_ROW_COLS.forEach(col => {
constantMap[col] = SQLiteType.TEXT
})
return {
const thisTable: SQLiteTable = {
...constantMap,
...fields,
}
tables[table._id!] = { fields: thisTable }
return tables
}
// nothing exists, need to iterate though existing tables
@ -60,8 +106,9 @@ async function buildBaseDefinition(): Promise<SQLiteDefinition> {
const tables = await tablesSdk.getAllInternalTables()
const definition = cloneDeep(BASIC_SQLITE_DOC)
for (let table of tables) {
definition.sql.tables[table._id!] = {
fields: mapTable(table),
definition.sql.tables = {
...definition.sql.tables,
...mapTable(table),
}
}
return definition
@ -75,8 +122,9 @@ export async function addTableToSqlite(table: Table) {
} catch (err) {
definition = await buildBaseDefinition()
}
definition.sql.tables[table._id!] = {
fields: mapTable(table),
definition.sql.tables = {
...definition.sql.tables,
...mapTable(table),
}
await db.put(definition)
}

4
packages/server/src/sdk/app/tables/tests/validation.spec.ts
View File

@ -125,7 +125,7 @@ describe("validation and update of external table schemas", () => {
}
it("should correctly set utilised foreign keys to autocolumns", () => {
const response = populateExternalTableSchemas(cloneDeep(SCHEMA) as any)
const response = populateExternalTableSchemas(cloneDeep(SCHEMA))
const foreignKey = getForeignKeyColumn(response)
expect(foreignKey.autocolumn).toBe(true)
expect(foreignKey.autoReason).toBe(AutoReason.FOREIGN_KEY)
@ -133,7 +133,7 @@ describe("validation and update of external table schemas", () => {
})
it("should correctly unset foreign keys when no longer used", () => {
const setResponse = populateExternalTableSchemas(cloneDeep(SCHEMA) as any)
const setResponse = populateExternalTableSchemas(cloneDeep(SCHEMA))
const beforeFk = getForeignKeyColumn(setResponse)
delete setResponse.entities!.client.schema.project
delete setResponse.entities!.project.schema.client

5
packages/server/src/sdk/app/tables/validation.ts
View File

@ -44,7 +44,10 @@ function checkForeignKeysAreAutoColumns(datasource: Datasource) {
if (shouldBeForeign && !column.autocolumn) {
column.autocolumn = true
column.autoReason = AutoReason.FOREIGN_KEY
} else if (column.autoReason === AutoReason.FOREIGN_KEY) {
} else if (
!shouldBeForeign &&
column.autoReason === AutoReason.FOREIGN_KEY
) {
delete column.autocolumn
delete column.autoReason
}

5
packages/server/src/sdk/app/views/tests/views.spec.ts
View File

@ -351,7 +351,6 @@ describe("table sdk", () => {
const view: ViewV2 = {
...basicView,
}
// eslint-disable-next-line @typescript-eslint/no-unused-vars
const { name, description, ...newTableSchema } = basicTable.schema
const result = syncSchema(_.cloneDeep(view), newTableSchema, undefined)
@ -365,7 +364,6 @@ describe("table sdk", () => {
const view: ViewV2 = {
...basicView,
}
// eslint-disable-next-line @typescript-eslint/no-unused-vars
const { description, ...newTableSchema } = {
...basicTable.schema,
updatedDescription: {
@ -450,7 +448,6 @@ describe("table sdk", () => {
hiddenField: { visible: false },
},
}
// eslint-disable-next-line @typescript-eslint/no-unused-vars
const { name, description, ...newTableSchema } = basicTable.schema
const result = syncSchema(_.cloneDeep(view), newTableSchema, undefined)
@ -474,7 +471,6 @@ describe("table sdk", () => {
hiddenField: { visible: false },
},
}
// eslint-disable-next-line @typescript-eslint/no-unused-vars
const { name, description, ...newTableSchema } = {
...basicTable.schema,
newField1: {
@ -506,7 +502,6 @@ describe("table sdk", () => {
hiddenField: { visible: false },
},
}
// eslint-disable-next-line @typescript-eslint/no-unused-vars
const { description, ...newTableSchema } = {
...basicTable.schema,
updatedDescription: {

15
packages/server/src/tests/utilities/api/datasource.ts
View File

@ -5,6 +5,7 @@ import {
UpdateDatasourceResponse,
UpdateDatasourceRequest,
QueryJson,
BuildSchemaFromSourceResponse,
} from "@budibase/types"
import { Expectations, TestAPI } from "./base"
@ -60,10 +61,22 @@ export class DatasourceAPI extends TestAPI {
})
}
query = async (query: QueryJson, expectations?: Expectations) => {
query = async (
query: Omit<QueryJson, "meta"> & Partial<Pick<QueryJson, "meta">>,
expectations?: Expectations
) => {
return await this._post<any>(`/api/datasources/query`, {
body: query,
expectations,
})
}
fetchSchema = async (id: string, expectations?: Expectations) => {
return await this._post<BuildSchemaFromSourceResponse>(
`/api/datasources/${id}/schema`,
{
expectations,
}
)
}
}

13
packages/server/src/threads/automation.ts
View File

@ -374,11 +374,13 @@ class Orchestrator {
for (let [innerObject, innerValue] of Object.entries(
originalStepInput[key][innerKey]
)) {
originalStepInput[key][innerKey][innerObject] =
automationUtils.substituteLoopStep(
innerValue as string,
`steps.${loopStepNumber}`
)
if (typeof innerValue === "string") {
originalStepInput[key][innerKey][innerObject] =
automationUtils.substituteLoopStep(
innerValue,
`steps.${loopStepNumber}`
)
}
}
}
}
@ -458,7 +460,6 @@ class Orchestrator {
inputs,
step.schema.inputs
)
try {
// appId is always passed
const outputs = await stepFn({

2
packages/server/src/utilities/fileSystem/filesystem.ts
View File

@ -43,7 +43,7 @@ export const checkDevelopmentEnvironment = () => {
error = "Must run via yarn once to generate environment."
}
if (error) {
console.error(error)
console.error("Error during development environment check", error)
process.exit(-1)
}
}

6
packages/server/src/websockets/websocket.ts
View File

@ -262,13 +262,11 @@ export class BaseSocket {
}
}
// eslint-disable-next-line @typescript-eslint/no-unused-vars
async onConnect(socket: Socket) {
async onConnect(_socket: Socket) {
// Override
}
// eslint-disable-next-line @typescript-eslint/no-unused-vars
async onDisconnect(socket: Socket) {
async onDisconnect(_socket: Socket) {
// Override
}

Some files were not shown because too many files have changed in this diff Show More