MirrorSave
/
budibase
mirror of https://github.com/Budibase/budibase.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Merge branch 'master' into fix-client-loading-issues

This commit is contained in:
Andrew Kingston 2024-04-19 10:53:23 +01:00 committed by GitHub
parent 6fa4eeb5c6 008c8e8c9d
commit c261ec10f4
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
41 changed files with 489 additions and 250 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
.vscode/settings.json vendored
View File

@ -24,5 +24,8 @@
},
"[svelte]": {
"editor.defaultFormatter": "svelte.svelte-vscode"
},
"[handlebars]": {
"editor.formatOnSave": false
}
}

2
lerna.json
View File

@ -1,5 +1,5 @@
{
"version": "2.23.6",
"version": "2.23.9",
"npmClient": "yarn",
"packages": [
"packages/*",

5
nx.json
View File

@ -9,10 +9,7 @@
},
"targetDefaults": {
"build": {
"inputs": [
"{workspaceRoot}/scripts/build.js",
"{workspaceRoot}/lerna.json"
]
"inputs": ["{workspaceRoot}/scripts/*", "{workspaceRoot}/lerna.json"]
}
}
}

2
package.json
View File

@ -59,7 +59,7 @@
"dev:camunda": "./scripts/deploy-camunda.sh",
"dev:all": "yarn run kill-all && lerna run --stream dev",
"dev:built": "yarn run kill-all && cd packages/server && yarn dev:stack:up && cd ../../ && lerna run --stream dev:built",
"dev:docker": "yarn build --scope @budibase/server --scope @budibase/worker && docker-compose -f hosting/docker-compose.build.yaml -f hosting/docker-compose.dev.yaml --env-file hosting/.env up --build --scale proxy-service=0",
"dev:docker": "./scripts/devDocker.sh",
"test": "REUSE_CONTAINERS=1 lerna run --concurrency 1 --stream test --stream",
"lint:eslint": "eslint packages --max-warnings=0",
"lint:prettier": "prettier --check \"packages/**/*.{js,ts,svelte}\" && prettier --write \"examples/**/*.{js,ts,svelte}\"",

4
packages/backend-core/src/users/db.ts
View File

@ -50,6 +50,8 @@ type CreateAdminUserOpts = {
hashPassword?: boolean
requirePassword?: boolean
skipPasswordValidation?: boolean
firstName?: string
lastName?: string
}
type FeatureFns = { isSSOEnforced: FeatureFn; isAppBuildersEnabled: FeatureFn }
@ -517,6 +519,8 @@ export class UserDB {
global: true,
},
tenantId,
firstName: opts?.firstName,
lastName: opts?.lastName,
}
if (opts?.ssoId) {
user.ssoId = opts.ssoId

52
packages/builder/src/components/backend/DataTable/modals/CreateEditColumn.svelte
View File

@ -13,6 +13,7 @@
Layout,
AbsTooltip,
} from "@budibase/bbui"
import { SWITCHABLE_TYPES, ValidColumnNameRegex } from "@budibase/shared-core"
import { createEventDispatcher, getContext, onMount } from "svelte"
import { cloneDeep } from "lodash/fp"
import { tables, datasources } from "stores/builder"
@ -20,11 +21,6 @@
import {
FIELDS,
RelationshipType,
ALLOWABLE_STRING_OPTIONS,
ALLOWABLE_NUMBER_OPTIONS,
ALLOWABLE_STRING_TYPES,
ALLOWABLE_NUMBER_TYPES,
SWITCHABLE_TYPES,
PrettyRelationshipDefinitions,
DB_TYPE_EXTERNAL,
} from "constants/backend"
@ -33,21 +29,20 @@
import ModalBindableInput from "components/common/bindings/ModalBindableInput.svelte"
import { getBindings } from "components/backend/DataTable/formula"
import JSONSchemaModal from "./JSONSchemaModal.svelte"
import { ValidColumnNameRegex } from "@budibase/shared-core"
import { FieldType, FieldSubtype, SourceName } from "@budibase/types"
import RelationshipSelector from "components/common/RelationshipSelector.svelte"
import { RowUtils } from "@budibase/frontend-core"
import ServerBindingPanel from "components/common/bindings/ServerBindingPanel.svelte"
const AUTO_TYPE = FIELDS.AUTO.type
const FORMULA_TYPE = FIELDS.FORMULA.type
const LINK_TYPE = FIELDS.LINK.type
const STRING_TYPE = FIELDS.STRING.type
const NUMBER_TYPE = FIELDS.NUMBER.type
const JSON_TYPE = FIELDS.JSON.type
const DATE_TYPE = FIELDS.DATETIME.type
const USER_TYPE = FIELDS.USER.subtype
const USERS_TYPE = FIELDS.USERS.subtype
const AUTO_TYPE = FieldType.AUTO
const FORMULA_TYPE = FieldType.FORMULA
const LINK_TYPE = FieldType.LINK
const STRING_TYPE = FieldType.STRING
const NUMBER_TYPE = FieldType.NUMBER
const JSON_TYPE = FieldType.JSON
const DATE_TYPE = FieldType.DATETIME
const USER_TYPE = FieldSubtype.USER
const USERS_TYPE = FieldSubtype.USERS
const dispatch = createEventDispatcher()
const PROHIBITED_COLUMN_NAMES = ["type", "_id", "_rev", "tableId"]
@ -61,8 +56,8 @@
let primaryDisplay
let indexes = [...($tables.selected.indexes || [])]
let isCreating = undefined
let relationshipPart1 = PrettyRelationshipDefinitions.Many
let relationshipPart2 = PrettyRelationshipDefinitions.One
let relationshipPart1 = PrettyRelationshipDefinitions.MANY
let relationshipPart2 = PrettyRelationshipDefinitions.ONE
let relationshipTableIdPrimary = null
let relationshipTableIdSecondary = null
let table = $tables.selected
@ -175,7 +170,7 @@
$: typeEnabled =
!originalName ||
(originalName &&
SWITCHABLE_TYPES.indexOf(editableColumn.type) !== -1 &&
SWITCHABLE_TYPES[field.type] &&
!editableColumn?.autocolumn)
const fieldDefinitions = Object.values(FIELDS).reduce(
@ -367,16 +362,15 @@
}
function getAllowedTypes() {
if (
originalName &&
ALLOWABLE_STRING_TYPES.indexOf(editableColumn.type) !== -1
) {
return ALLOWABLE_STRING_OPTIONS
} else if (
originalName &&
ALLOWABLE_NUMBER_TYPES.indexOf(editableColumn.type) !== -1
) {
return ALLOWABLE_NUMBER_OPTIONS
if (originalName) {
const possibleTypes = (
SWITCHABLE_TYPES[field.type] || [editableColumn.type]
).map(t => t.toLowerCase())
return Object.entries(FIELDS)
.filter(([fieldType]) =>
possibleTypes.includes(fieldType.toLowerCase())
)
.map(([_, fieldDefinition]) => fieldDefinition)
}
const isUsers =
@ -632,7 +626,7 @@
/>
</div>
</div>
{:else if editableColumn.type === FieldType.LINK}
{:else if editableColumn.type === FieldType.LINK && !editableColumn.autocolumn}
<RelationshipSelector
bind:relationshipPart1
bind:relationshipPart2

20
packages/builder/src/constants/backend/index.js
View File

@ -202,26 +202,6 @@ export const PrettyRelationshipDefinitions = {
ONE: "One row",
}
export const ALLOWABLE_STRING_OPTIONS = [
FIELDS.STRING,
FIELDS.OPTIONS,
FIELDS.LONGFORM,
FIELDS.BARCODEQR,
]
export const ALLOWABLE_STRING_TYPES = ALLOWABLE_STRING_OPTIONS.map(
opt => opt.type
)
export const ALLOWABLE_NUMBER_OPTIONS = [FIELDS.NUMBER, FIELDS.BOOLEAN]
export const ALLOWABLE_NUMBER_TYPES = ALLOWABLE_NUMBER_OPTIONS.map(
opt => opt.type
)
export const SWITCHABLE_TYPES = [
...ALLOWABLE_STRING_TYPES,
...ALLOWABLE_NUMBER_TYPES,
]
export const BUDIBASE_INTERNAL_DB_ID = INTERNAL_TABLE_SOURCE_ID
export const DEFAULT_BB_DATASOURCE_ID = "datasource_internal_bb_default"
export const BUDIBASE_DATASOURCE_TYPE = "budibase"

10
packages/builder/src/stores/builder/tables.js
View File

@ -1,8 +1,8 @@
import { FieldType } from "@budibase/types"
import { SWITCHABLE_TYPES } from "@budibase/shared-core"
import { get, writable, derived } from "svelte/store"
import { cloneDeep } from "lodash/fp"
import { API } from "api"
import { SWITCHABLE_TYPES } from "constants/backend"
export function createTablesStore() {
const store = writable({
@ -64,7 +64,7 @@ export function createTablesStore() {
if (
oldField != null &&
oldField?.type !== field.type &&
SWITCHABLE_TYPES.indexOf(oldField?.type) === -1
!SWITCHABLE_TYPES[oldField?.type]?.includes(field.type)
) {
updatedTable.schema[key] = oldField
}
@ -148,12 +148,6 @@ export function createTablesStore() {
if (indexes) {
draft.indexes = indexes
}
// Add object to indicate if column is being added
if (draft.schema[field.name] === undefined) {
draft._add = {
name: field.name,
}
}
draft.schema = {
...draft.schema,
[field.name]: cloneDeep(field),

1
packages/server/scripts/integrations/mssql/docker-compose.yaml
View File

@ -4,6 +4,7 @@ services:
# user: sa
# database: master
mssql:
# platform: linux/amd64
image: bb/mssql
build:
context: .

29
packages/server/src/api/controllers/row/ExternalRequest.ts
View File

@ -374,38 +374,44 @@ export class ExternalRequest<T extends Operation> {
) {
continue
}
let tableId: string | undefined,
let relatedTableId: string | undefined,
lookupField: string | undefined,
fieldName: string | undefined
if (isManyToMany(field)) {
tableId = field.through
relatedTableId = field.through
lookupField = primaryKey
fieldName = field.throughTo || primaryKey
} else if (isManyToOne(field)) {
tableId = field.tableId
relatedTableId = field.tableId
lookupField = field.foreignKey
fieldName = field.fieldName
}
if (!tableId || !lookupField || !fieldName) {
if (!relatedTableId || !lookupField || !fieldName) {
throw new Error(
"Unable to lookup relationships - undefined column properties."
)
}
const { tableName: relatedTableName } = breakExternalTableId(tableId)
const { tableName: relatedTableName } =
breakExternalTableId(relatedTableId)
// @ts-ignore
const linkPrimaryKey = this.tables[relatedTableName].primary[0]
if (!lookupField || !row[lookupField]) {
continue
}
const endpoint = getEndpoint(relatedTableId, Operation.READ)
const relatedTable = this.tables[endpoint.entityId]
if (!relatedTable) {
throw new Error("unable to find related table")
}
const response = await getDatasourceAndQuery({
endpoint: getEndpoint(tableId, Operation.READ),
endpoint: endpoint,
filters: {
equal: {
[fieldName]: row[lookupField],
},
},
meta: {
table,
table: relatedTable,
},
})
// this is the response from knex if no rows found
@ -414,7 +420,11 @@ export class ExternalRequest<T extends Operation> {
const storeTo = isManyToMany(field)
? field.throughFrom || linkPrimaryKey
: fieldName
related[storeTo] = { rows, isMany: isManyToMany(field), tableId }
related[storeTo] = {
rows,
isMany: isManyToMany(field),
tableId: relatedTableId,
}
}
return related
}
@ -437,7 +447,6 @@ export class ExternalRequest<T extends Operation> {
// if we're creating (in a through table) need to wipe the existing ones first
const promises = []
const related = await this.lookupRelations(mainTableId, row)
const table = this.getTable(mainTableId)!
for (let relationship of relationships) {
const { key, tableId, isUpdate, id, ...rest } = relationship
const body: { [key: string]: any } = processObjectSync(rest, row, {})
@ -484,7 +493,7 @@ export class ExternalRequest<T extends Operation> {
body,
filters: buildFilters(id, {}, linkTable),
meta: {
table,
table: linkTable,
},
})
)

3
packages/server/src/api/controllers/table/external.ts
View File

@ -31,7 +31,6 @@ export async function save(
renaming?: RenameColumn
) {
const inputs = ctx.request.body
const adding = inputs?._add
// can't do this right now
delete inputs.rows
const tableId = ctx.request.body._id
@ -44,7 +43,7 @@ export async function save(
const { datasource, table } = await sdk.tables.external.save(
datasourceId!,
inputs,
{ tableId, renaming, adding }
{ tableId, renaming }
)
builderSocket?.emitDatasourceUpdate(ctx, datasource)
return table

5
packages/server/src/api/controllers/table/index.ts
View File

@ -77,11 +77,6 @@ export async function save(ctx: UserCtx<SaveTableRequest, SaveTableResponse>) {
const renaming = ctx.request.body._rename
const api = pickApi({ table })
// do not pass _rename or _add if saving to CouchDB
if (api === internal) {
delete ctx.request.body._add
delete ctx.request.body._rename
}
let savedTable = await api.save(ctx, renaming)
if (!table._id) {
savedTable = sdk.tables.enrichViewSchemas(savedTable)

2
packages/server/src/api/controllers/table/internal.ts
View File

@ -16,7 +16,7 @@ export async function save(
ctx: UserCtx<SaveTableRequest, SaveTableResponse>,
renaming?: RenameColumn
) {
const { rows, ...rest } = ctx.request.body
const { _rename, rows, ...rest } = ctx.request.body
let tableToSave: Table = {
_id: generateTableID(),
...rest,

18
packages/server/src/api/routes/tests/queries/generic-sql.spec.ts
View File

@ -4,6 +4,7 @@ import {
Query,
QueryPreview,
SourceName,
TableSourceType,
} from "@budibase/types"
import * as setup from "../utilities"
import {
@ -740,12 +741,25 @@ describe.each(
})
describe("query through datasource", () => {
it("should be able to query a pg datasource", async () => {
it("should be able to query the datasource", async () => {
const entityId = "test_table"
await config.api.datasource.update({
...datasource,
entities: {
[entityId]: {
name: entityId,
schema: {},
type: "table",
sourceId: datasource._id!,
sourceType: TableSourceType.EXTERNAL,
},
},
})
const res = await config.api.datasource.query({
endpoint: {
datasourceId: datasource._id!,
operation: Operation.READ,
entityId: "test_table",
entityId,
},
resource: {
fields: ["id", "name"],

125
packages/server/src/api/routes/tests/search.spec.ts
View File

@ -26,6 +26,7 @@ describe.each([
[DatabaseName.MARIADB, getDatasource(DatabaseName.MARIADB)],
])("/api/:sourceId/search (%s)", (name, dsProvider) => {
const isSqs = name === "internal-sqs"
const isInternal = name === "internal"
const config = setup.getConfig()
let envCleanup: (() => void) | undefined
@ -336,6 +337,20 @@ describe.each([
expectQuery({
range: { age: { low: 5, high: 9 } },
}).toFindNothing())
// We never implemented half-open ranges in Lucene.
!isInternal &&
it("can search using just a low value", () =>
expectQuery({
range: { age: { low: 5 } },
}).toContainExactly([{ age: 10 }]))
// We never implemented half-open ranges in Lucene.
!isInternal &&
it("can search using just a high value", () =>
expectQuery({
range: { age: { high: 5 } },
}).toContainExactly([{ age: 1 }]))
})
describe("sort", () => {
@ -440,6 +455,20 @@ describe.each([
expectQuery({
range: { dob: { low: JAN_5TH, high: JAN_9TH } },
}).toFindNothing())
// We never implemented half-open ranges in Lucene.
!isInternal &&
it("can search using just a low value", () =>
expectQuery({
range: { dob: { low: JAN_5TH } },
}).toContainExactly([{ dob: JAN_10TH }]))
// We never implemented half-open ranges in Lucene.
!isInternal &&
it("can search using just a high value", () =>
expectQuery({
range: { dob: { high: JAN_5TH } },
}).toContainExactly([{ dob: JAN_1ST }]))
})
describe("sort", () => {
@ -550,4 +579,100 @@ describe.each([
]))
})
})
describe("bigints", () => {
const SMALL = "1"
const MEDIUM = "10000000"
// Our bigints are int64s in most datasources.
const BIG = "9223372036854775807"
beforeAll(async () => {
await createTable({
num: { name: "num", type: FieldType.BIGINT },
})
await createRows([{ num: SMALL }, { num: MEDIUM }, { num: BIG }])
})
describe("equal", () => {
it("successfully finds a row", () =>
expectQuery({ equal: { num: SMALL } }).toContainExactly([
{ num: SMALL },
]))
it("successfully finds a big value", () =>
expectQuery({ equal: { num: BIG } }).toContainExactly([{ num: BIG }]))
it("fails to find nonexistent row", () =>
expectQuery({ equal: { num: "2" } }).toFindNothing())
})
describe("notEqual", () => {
it("successfully finds a row", () =>
expectQuery({ notEqual: { num: SMALL } }).toContainExactly([
{ num: MEDIUM },
{ num: BIG },
]))
it("fails to find nonexistent row", () =>
expectQuery({ notEqual: { num: 10 } }).toContainExactly([
{ num: SMALL },
{ num: MEDIUM },
{ num: BIG },
]))
})
describe("oneOf", () => {
it("successfully finds a row", () =>
expectQuery({ oneOf: { num: [SMALL] } }).toContainExactly([
{ num: SMALL },
]))
it("successfully finds all rows", () =>
expectQuery({ oneOf: { num: [SMALL, MEDIUM, BIG] } }).toContainExactly([
{ num: SMALL },
{ num: MEDIUM },
{ num: BIG },
]))
it("fails to find nonexistent row", () =>
expectQuery({ oneOf: { num: [2] } }).toFindNothing())
})
// Range searches against bigints don't seem to work at all in Lucene, and I
// couldn't figure out why. Given that we're replacing Lucene with SQS,
// we've decided not to spend time on it.
!isInternal &&
describe("range", () => {
it("successfully finds a row", () =>
expectQuery({
range: { num: { low: SMALL, high: "5" } },
}).toContainExactly([{ num: SMALL }]))
it("successfully finds multiple rows", () =>
expectQuery({
range: { num: { low: SMALL, high: MEDIUM } },
}).toContainExactly([{ num: SMALL }, { num: MEDIUM }]))
it("successfully finds a row with a high bound", () =>
expectQuery({
range: { num: { low: MEDIUM, high: BIG } },
}).toContainExactly([{ num: MEDIUM }, { num: BIG }]))
it("successfully finds no rows", () =>
expectQuery({
range: { num: { low: "5", high: "5" } },
}).toFindNothing())
it("can search using just a low value", () =>
expectQuery({
range: { num: { low: MEDIUM } },
}).toContainExactly([{ num: MEDIUM }, { num: BIG }]))
it("can search using just a high value", () =>
expectQuery({
range: { num: { high: MEDIUM } },
}).toContainExactly([{ num: SMALL }, { num: MEDIUM }]))
})
})
})

4
packages/server/src/api/routes/tests/table.spec.ts
View File

@ -219,9 +219,6 @@ describe.each([
it("should add a new column for an internal DB table", async () => {
const saveTableRequest: SaveTableRequest = {
_add: {
name: "NEW_COLUMN",
},
...basicTable(),
}
@ -235,7 +232,6 @@ describe.each([
updatedAt: expect.stringMatching(ISO_REGEX_PATTERN),
views: {},
}
delete expectedResponse._add
expect(response).toEqual(expectedResponse)
})
})

67
packages/server/src/integration-test/mysql.spec.ts
View File

@ -16,7 +16,6 @@ import {
getDatasource,
rawQuery,
} from "../integrations/tests/utils"
import { builderSocket } from "../websockets"
import { generator } from "@budibase/backend-core/tests"
// @ts-ignore
fetch.mockSearch()
@ -233,72 +232,6 @@ describe("mysql integrations", () => {
})
describe("POST /api/tables/", () => {
const emitDatasourceUpdateMock = jest.fn()
it("will emit the datasource entity schema with externalType to the front-end when adding a new column", async () => {
const addColumnToTable: TableRequest = {
type: "table",
sourceType: TableSourceType.EXTERNAL,
name: uniqueTableName(),
sourceId: datasource._id!,
primary: ["id"],
schema: {
id: {
type: FieldType.AUTO,
name: "id",
autocolumn: true,
},
new_column: {
type: FieldType.NUMBER,
name: "new_column",
},
},
_add: {
name: "new_column",
},
}
jest
.spyOn(builderSocket!, "emitDatasourceUpdate")
.mockImplementation(emitDatasourceUpdateMock)
await makeRequest("post", "/api/tables/", addColumnToTable)
const expectedTable: TableRequest = {
...addColumnToTable,
schema: {
id: {
type: FieldType.NUMBER,
name: "id",
autocolumn: true,
constraints: {
presence: false,
},
externalType: "int unsigned",
},
new_column: {
type: FieldType.NUMBER,
name: "new_column",
autocolumn: false,
constraints: {
presence: false,
},
externalType: "float(8,2)",
},
},
created: true,
_id: `${datasource._id}__${addColumnToTable.name}`,
}
delete expectedTable._add
expect(emitDatasourceUpdateMock).toHaveBeenCalledTimes(1)
const emittedDatasource: Datasource =
emitDatasourceUpdateMock.mock.calls[0][1]
expect(emittedDatasource.entities![expectedTable.name]).toEqual(
expectedTable
)
})
it("will rename a column", async () => {
await makeRequest("post", "/api/tables/", primaryMySqlTable)

12
packages/server/src/integrations/base/query.ts
View File

@ -2,6 +2,7 @@ import {
QueryJson,
Datasource,
DatasourcePlusQueryResponse,
RowOperations,
} from "@budibase/types"
import { getIntegration } from "../index"
import sdk from "../../sdk"
@ -10,6 +11,17 @@ export async function makeExternalQuery(
datasource: Datasource,
json: QueryJson
): Promise<DatasourcePlusQueryResponse> {
const entityId = json.endpoint.entityId,
tableName = json.meta.table.name,
tableId = json.meta.table._id
// case found during testing - make sure this doesn't happen again
if (
RowOperations.includes(json.endpoint.operation) &&
entityId !== tableId &&
entityId !== tableName
) {
throw new Error("Entity ID and table metadata do not align")
}
datasource = await sdk.datasources.enrich(datasource)
const Integration = await getIntegration(datasource.source)
// query is the opinionated function

58
packages/server/src/integrations/base/sql.ts
View File

@ -6,6 +6,7 @@ import {
SqlClient,
isValidFilter,
getNativeSql,
SqlStatements,
} from "../utils"
import SqlTableQueryBuilder from "./sqlTable"
import {
@ -160,9 +161,19 @@ class InternalBuilder {
addFilters(
query: Knex.QueryBuilder,
filters: SearchFilters | undefined,
tableName: string,
table: Table,
opts: { aliases?: Record<string, string>; relationship?: boolean }
): Knex.QueryBuilder {
if (!filters) {
return query
}
filters = parseFilters(filters)
// if all or specified in filters, then everything is an or
const allOr = filters.allOr
const sqlStatements = new SqlStatements(this.client, table, { allOr })
const tableName =
this.client === SqlClient.SQL_LITE ? table._id! : table.name
function getTableAlias(name: string) {
const alias = opts.aliases?.[name]
return alias || name
@ -258,12 +269,6 @@ class InternalBuilder {
}
}
if (!filters) {
return query
}
filters = parseFilters(filters)
// if all or specified in filters, then everything is an or
const allOr = filters.allOr
if (filters.oneOf) {
iterate(filters.oneOf, (key, array) => {
const fnc = allOr ? "orWhereIn" : "whereIn"
@ -306,17 +311,11 @@ class InternalBuilder {
const lowValid = isValidFilter(value.low),
highValid = isValidFilter(value.high)
if (lowValid && highValid) {
// Use a between operator if we have 2 valid range values
const fnc = allOr ? "orWhereBetween" : "whereBetween"
query = query[fnc](key, [value.low, value.high])
query = sqlStatements.between(query, key, value.low, value.high)
} else if (lowValid) {
// Use just a single greater than operator if we only have a low
const fnc = allOr ? "orWhere" : "where"
query = query[fnc](key, ">", value.low)
query = sqlStatements.lte(query, key, value.low)
} else if (highValid) {
// Use just a single less than operator if we only have a high
const fnc = allOr ? "orWhere" : "where"
query = query[fnc](key, "<", value.high)
query = sqlStatements.gte(query, key, value.high)
}
})
}
@ -359,7 +358,7 @@ class InternalBuilder {
addSorting(query: Knex.QueryBuilder, json: QueryJson): Knex.QueryBuilder {
let { sort, paginate } = json
const table = json.meta?.table
const table = json.meta.table
const tableName = getTableName(table)
const aliases = json.tableAliases
const aliased =
@ -473,14 +472,13 @@ class InternalBuilder {
): Knex.QueryBuilder {
const tableName = endpoint.entityId
const tableAlias = aliases?.[tableName]
let table: string | Record<string, string> = tableName
if (tableAlias) {
table = { [tableAlias]: tableName }
}
let query = knex(table)
if (endpoint.schema) {
query = query.withSchema(endpoint.schema)
}
const query = knex(
this.tableNameWithSchema(tableName, {
alias: tableAlias,
schema: endpoint.schema,
})
)
return query
}
@ -547,7 +545,7 @@ class InternalBuilder {
if (foundOffset) {
query = query.offset(foundOffset)
}
query = this.addFilters(query, filters, tableName, {
query = this.addFilters(query, filters, json.meta.table, {
aliases: tableAliases,
})
// add sorting to pre-query
@ -568,7 +566,7 @@ class InternalBuilder {
endpoint.schema,
tableAliases
)
return this.addFilters(query, filters, tableName, {
return this.addFilters(query, filters, json.meta.table, {
relationship: true,
aliases: tableAliases,
})
@ -578,7 +576,7 @@ class InternalBuilder {
const { endpoint, body, filters, tableAliases } = json
let query = this.knexWithAlias(knex, endpoint, tableAliases)
const parsedBody = parseBody(body)
query = this.addFilters(query, filters, endpoint.entityId, {
query = this.addFilters(query, filters, json.meta.table, {
aliases: tableAliases,
})
// mysql can't use returning
@ -592,7 +590,7 @@ class InternalBuilder {
delete(knex: Knex, json: QueryJson, opts: QueryOptions): Knex.QueryBuilder {
const { endpoint, filters, tableAliases } = json
let query = this.knexWithAlias(knex, endpoint, tableAliases)
query = this.addFilters(query, filters, endpoint.entityId, {
query = this.addFilters(query, filters, json.meta.table, {
aliases: tableAliases,
})
// mysql can't use returning
@ -684,7 +682,7 @@ class SqlQueryBuilder extends SqlTableQueryBuilder {
// when creating if an ID has been inserted need to make sure
// the id filter is enriched with it before trying to retrieve the row
checkLookupKeys(id: any, json: QueryJson) {
if (!id || !json.meta?.table || !json.meta.table.primary) {
if (!id || !json.meta.table || !json.meta.table.primary) {
return json
}
const primaryKey = json.meta.table.primary?.[0]

24
packages/server/src/integrations/tests/sql.spec.ts
View File

@ -1,11 +1,11 @@
import { SqlClient } from "../utils"
import Sql from "../base/sql"
import {
FieldType,
Operation,
QueryJson,
TableSourceType,
Table,
FieldType,
TableSourceType,
} from "@budibase/types"
const TABLE_NAME = "test"
@ -13,7 +13,12 @@ const TABLE: Table = {
type: "table",
sourceType: TableSourceType.EXTERNAL,
sourceId: "SOURCE_ID",
schema: {},
schema: {
id: {
name: "id",
type: FieldType.NUMBER,
},
},
name: TABLE_NAME,
primary: ["id"],
}
@ -73,7 +78,7 @@ function generateUpdateJson({
meta?: any
}): QueryJson {
if (!meta.table) {
meta.table = table
meta.table = TABLE
}
return {
endpoint: endpoint(table, "UPDATE"),
@ -158,6 +163,9 @@ function generateManyRelationshipJson(config: { schema?: string } = {}) {
},
],
extra: { idFilter: {} },
meta: {
table: TABLE,
},
}
}
@ -341,7 +349,7 @@ describe("SQL query builder", () => {
)
expect(query).toEqual({
bindings: [date, limit],
sql: `select * from (select * from "${TABLE_NAME}" where "${TABLE_NAME}"."property" > $1 limit $2) as "${TABLE_NAME}"`,
sql: `select * from (select * from "${TABLE_NAME}" where "${TABLE_NAME}"."property" >= $1 limit $2) as "${TABLE_NAME}"`,
})
})
@ -360,7 +368,7 @@ describe("SQL query builder", () => {
)
expect(query).toEqual({
bindings: [date, limit],
sql: `select * from (select * from "${TABLE_NAME}" where "${TABLE_NAME}"."property" < $1 limit $2) as "${TABLE_NAME}"`,
sql: `select * from (select * from "${TABLE_NAME}" where "${TABLE_NAME}"."property" <= $1 limit $2) as "${TABLE_NAME}"`,
})
})
@ -594,7 +602,7 @@ describe("SQL query builder", () => {
)
expect(query).toEqual({
bindings: ["2000-01-01 00:00:00", 500],
sql: `select * from (select * from "${TABLE_NAME}" where "${TABLE_NAME}"."dob" > $1 limit $2) as "${TABLE_NAME}"`,
sql: `select * from (select * from "${TABLE_NAME}" where "${TABLE_NAME}"."dob" >= $1 limit $2) as "${TABLE_NAME}"`,
})
})
@ -613,7 +621,7 @@ describe("SQL query builder", () => {
)
expect(query).toEqual({
bindings: ["2010-01-01 00:00:00", 500],
sql: `select * from (select * from "${TABLE_NAME}" where "${TABLE_NAME}"."dob" < $1 limit $2) as "${TABLE_NAME}"`,
sql: `select * from (select * from "${TABLE_NAME}" where "${TABLE_NAME}"."dob" <= $1 limit $2) as "${TABLE_NAME}"`,
})
})

3
packages/server/src/integrations/tests/sqlAlias.spec.ts
View File

@ -117,7 +117,8 @@ describe("Captures of real examples", () => {
let query = new Sql(SqlClient.POSTGRES, limit)._query(queryJson)
const filters = queryJson.filters
const notEqualsValue = Object.values(filters?.notEqual!)[0]
const rangeValue = Object.values(filters?.range!)[0]
const rangeValue: { high?: string | number; low?: string | number } =
Object.values(filters?.range!)[0]
const equalValue = Object.values(filters?.equal!)[0]
expect(query).toEqual({

2
packages/server/src/integrations/utils/index.ts Normal file
View File

@ -0,0 +1,2 @@
export * from "./utils"
export { SqlStatements } from "./sqlStatements"

80
packages/server/src/integrations/utils/sqlStatements.ts Normal file
View File

@ -0,0 +1,80 @@
import { FieldType, Table, FieldSchema } from "@budibase/types"
import { SqlClient } from "./utils"
import { Knex } from "knex"
export class SqlStatements {
client: string
table: Table
allOr: boolean | undefined
constructor(
client: string,
table: Table,
{ allOr }: { allOr?: boolean } = {}
) {
this.client = client
this.table = table
this.allOr = allOr
}
getField(key: string): FieldSchema | undefined {
const fieldName = key.split(".")[1]
return this.table.schema[fieldName]
}
between(
query: Knex.QueryBuilder,
key: string,
low: number | string,
high: number | string
) {
// Use a between operator if we have 2 valid range values
const field = this.getField(key)
if (
field?.type === FieldType.BIGINT &&
this.client === SqlClient.SQL_LITE
) {
query = query.whereRaw(
`CAST(${key} AS INTEGER) BETWEEN CAST(? AS INTEGER) AND CAST(? AS INTEGER)`,
[low, high]
)
} else {
const fnc = this.allOr ? "orWhereBetween" : "whereBetween"
query = query[fnc](key, [low, high])
}
return query
}
lte(query: Knex.QueryBuilder, key: string, low: number | string) {
// Use just a single greater than operator if we only have a low
const field = this.getField(key)
if (
field?.type === FieldType.BIGINT &&
this.client === SqlClient.SQL_LITE
) {
query = query.whereRaw(`CAST(${key} AS INTEGER) >= CAST(? AS INTEGER)`, [
low,
])
} else {
const fnc = this.allOr ? "orWhere" : "where"
query = query[fnc](key, ">=", low)
}
return query
}
gte(query: Knex.QueryBuilder, key: string, high: number | string) {
const field = this.getField(key)
// Use just a single less than operator if we only have a high
if (
field?.type === FieldType.BIGINT &&
this.client === SqlClient.SQL_LITE
) {
query = query.whereRaw(`CAST(${key} AS INTEGER) <= CAST(? AS INTEGER)`, [
high,
])
} else {
const fnc = this.allOr ? "orWhere" : "where"
query = query[fnc](key, "<=", high)
}
return query
}
}

69
packages/server/src/integrations/utils.ts → packages/server/src/integrations/utils/utils.ts
View File

@ -5,10 +5,10 @@ import {
FieldType,
TableSourceType,
} from "@budibase/types"
import { DocumentType, SEPARATOR } from "../db/utils"
import { InvalidColumns, DEFAULT_BB_DATASOURCE_ID } from "../constants"
import { helpers } from "@budibase/shared-core"
import env from "../environment"
import { DocumentType, SEPARATOR } from "../../db/utils"
import { InvalidColumns, DEFAULT_BB_DATASOURCE_ID } from "../../constants"
import { SWITCHABLE_TYPES, helpers } from "@budibase/shared-core"
import env from "../../environment"
import { Knex } from "knex"
const DOUBLE_SEPARATOR = `${SEPARATOR}${SEPARATOR}`
@ -284,8 +284,8 @@ export function isIsoDateString(str: string) {
* @param column The column to check, to see if it is a valid relationship.
* @param tableIds The IDs of the tables which currently exist.
*/
export function shouldCopyRelationship(
column: { type: string; tableId?: string },
function shouldCopyRelationship(
column: { type: FieldType.LINK; tableId?: string },
tableIds: string[]
) {
return (
@ -303,28 +303,18 @@ export function shouldCopyRelationship(
* @param column The column to check for options or boolean type.
* @param fetchedColumn The fetched column to check for the type in the external database.
*/
export function shouldCopySpecialColumn(
column: { type: string },
fetchedColumn: { type: string } | undefined
function shouldCopySpecialColumn(
column: { type: FieldType },
fetchedColumn: { type: FieldType } | undefined
) {
const isFormula = column.type === FieldType.FORMULA
const specialTypes = [
FieldType.OPTIONS,
FieldType.LONGFORM,
FieldType.ARRAY,
FieldType.FORMULA,
FieldType.BB_REFERENCE,
]
// column has been deleted, remove - formulas will never exist, always copy
if (!isFormula && column && !fetchedColumn) {
return false
}
const fetchedIsNumber =
!fetchedColumn || fetchedColumn.type === FieldType.NUMBER
return (
specialTypes.indexOf(column.type as FieldType) !== -1 ||
(fetchedIsNumber && column.type === FieldType.BOOLEAN)
)
return fetchedIsNumber && column.type === FieldType.BOOLEAN
}
/**
@ -357,11 +347,44 @@ function copyExistingPropsOver(
continue
}
const column = existingTableSchema[key]
const existingColumnType = column?.type
const updatedColumnType = table.schema[key]?.type
// If the db column type changed to a non-compatible one, we want to re-fetch it
if (
shouldCopyRelationship(column, tableIds) ||
shouldCopySpecialColumn(column, table.schema[key])
updatedColumnType !== existingColumnType &&
!SWITCHABLE_TYPES[updatedColumnType]?.includes(existingColumnType)
) {
table.schema[key] = existingTableSchema[key]
continue
}
if (
column.type === FieldType.LINK &&
!shouldCopyRelationship(column, tableIds)
) {
continue
}
const specialTypes = [
FieldType.OPTIONS,
FieldType.LONGFORM,
FieldType.ARRAY,
FieldType.FORMULA,
FieldType.BB_REFERENCE,
]
if (
specialTypes.includes(column.type) &&
!shouldCopySpecialColumn(column, table.schema[key])
) {
continue
}
table.schema[key] = {
...existingTableSchema[key],
externalType:
existingTableSchema[key].externalType ||
table.schema[key].externalType,
}
}
}

3
packages/server/src/sdk/app/datasources/datasources.ts
View File

@ -348,8 +348,7 @@ const preSaveAction: Partial<Record<SourceName, any>> = {
* Make sure all datasource entities have a display name selected
*/
export function setDefaultDisplayColumns(datasource: Datasource) {
//
for (let entity of Object.values(datasource.entities || {})) {
for (const entity of Object.values(datasource.entities || {})) {
if (entity.primaryDisplay) {
continue
}

2
packages/server/src/sdk/app/rows/search/sqs.ts
View File

@ -200,6 +200,6 @@ export async function search(
}
} catch (err: any) {
const msg = typeof err === "string" ? err : err.message
throw new Error(`Unable to search by SQL - ${msg}`)
throw new Error(`Unable to search by SQL - ${msg}`, { cause: err })
}
}

6
packages/server/src/sdk/app/rows/utils.ts
View File

@ -52,6 +52,12 @@ export async function getDatasourceAndQuery(
): Promise<DatasourcePlusQueryResponse> {
const datasourceId = json.endpoint.datasourceId
const datasource = await sdk.datasources.get(datasourceId)
const table = datasource.entities?.[json.endpoint.entityId]
if (!json.meta && table) {
json.meta = {
table,
}
}
return makeExternalQuery(datasource, json)
}

16
packages/server/src/sdk/app/tables/external/index.ts vendored
View File

@ -3,7 +3,6 @@ import {
Operation,
RelationshipType,
RenameColumn,
AddColumn,
Table,
TableRequest,
ViewV2,
@ -33,7 +32,7 @@ import * as viewSdk from "../../views"
export async function save(
datasourceId: string,
update: Table,
opts?: { tableId?: string; renaming?: RenameColumn; adding?: AddColumn }
opts?: { tableId?: string; renaming?: RenameColumn }
) {
let tableToSave: TableRequest = {
...update,
@ -52,6 +51,12 @@ export async function save(
!oldTable &&
(tableToSave.primary == null || tableToSave.primary.length === 0)
) {
if (tableToSave.schema.id) {
throw new Error(
"External tables with no `primary` column set will define an `id` column, but we found an `id` column in the supplied schema. Either set a `primary` column or remove the `id` column."
)
}
tableToSave.primary = ["id"]
tableToSave.schema.id = {
type: FieldType.NUMBER,
@ -179,14 +184,7 @@ export async function save(
// remove the rename prop
delete tableToSave._rename
// if adding a new column, we need to rebuild the schema for that table to get the 'externalType' of the column
if (opts?.adding) {
datasource.entities[tableToSave.name] = (
await datasourceSdk.buildFilteredSchema(datasource, [tableToSave.name])
).tables[tableToSave.name]
} else {
datasource.entities[tableToSave.name] = tableToSave
}
// store it into couch now for budibase reference
await db.put(populateExternalTableSchemas(datasource))

2
packages/server/src/sdk/app/tables/internal/sqs.ts
View File

@ -42,7 +42,7 @@ const FieldTypeMap: Record<FieldType, SQLiteType> = {
[FieldType.ATTACHMENT_SINGLE]: SQLiteType.BLOB,
[FieldType.ARRAY]: SQLiteType.BLOB,
[FieldType.LINK]: SQLiteType.BLOB,
[FieldType.BIGINT]: SQLiteType.REAL,
[FieldType.BIGINT]: SQLiteType.TEXT,
// TODO: consider the difference between multi-user and single user types (subtyping)
[FieldType.BB_REFERENCE]: SQLiteType.TEXT,
}

2
packages/server/src/tests/utilities/api/datasource.ts
View File

@ -61,7 +61,7 @@ export class DatasourceAPI extends TestAPI {
}
query = async (
query: Omit<QueryJson, "meta">,
query: Omit<QueryJson, "meta"> & Partial<Pick<QueryJson, "meta">>,
expectations?: Expectations
) => {
return await this._post<any>(`/api/datasources/query`, {

33
packages/shared-core/src/constants/fields.ts Normal file
View File

@ -0,0 +1,33 @@
import { FieldType } from "@budibase/types"
type SwitchableTypes = Partial<{
[K in FieldType]: [K, ...FieldType[]]
}>
export const SWITCHABLE_TYPES: SwitchableTypes = {
[FieldType.STRING]: [
FieldType.STRING,
FieldType.OPTIONS,
FieldType.LONGFORM,
FieldType.BARCODEQR,
],
[FieldType.OPTIONS]: [
FieldType.OPTIONS,
FieldType.STRING,
FieldType.LONGFORM,
FieldType.BARCODEQR,
],
[FieldType.LONGFORM]: [
FieldType.LONGFORM,
FieldType.STRING,
FieldType.OPTIONS,
FieldType.BARCODEQR,
],
[FieldType.BARCODEQR]: [
FieldType.BARCODEQR,
FieldType.STRING,
FieldType.OPTIONS,
FieldType.LONGFORM,
],
[FieldType.NUMBER]: [FieldType.NUMBER, FieldType.BOOLEAN],
}

1
packages/shared-core/src/constants/index.ts
View File

@ -1,4 +1,5 @@
export * from "./api"
export * from "./fields"
export const OperatorOptions = {
Equals: {

18
packages/shared-core/src/filters.ts
View File

@ -218,14 +218,16 @@ export const buildLuceneQuery = (filter: SearchFilter[]) => {
high: type === "number" ? maxint : "9999-00-00T00:00:00.000Z",
}
}
if ((operator as any) === "rangeLow" && value != null && value !== "") {
query.range[field].low = value
} else if (
(operator as any) === "rangeHigh" &&
value != null &&
value !== ""
) {
query.range[field].high = value
if (operator === "rangeLow" && value != null && value !== "") {
query.range[field] = {
...query.range[field],
low: value,
}
} else if (operator === "rangeHigh" && value != null && value !== "") {
query.range[field] = {
...query.range[field],
high: value,
}
}
} else if (query[queryOperator] && operator !== "onEmptyFilter") {
if (type === "boolean") {

2
packages/types/src/api/web/user.ts
View File

@ -66,6 +66,8 @@ export interface CreateAdminUserRequest {
password?: string
tenantId: string
ssoId?: string
familyName?: string
givenName?: string
}
export interface AddSSoUserRequest {

3
packages/types/src/documents/app/table/table.ts
View File

@ -1,6 +1,6 @@
import { Document } from "../../document"
import { View, ViewV2 } from "../view"
import { AddColumn, RenameColumn } from "../../../sdk"
import { RenameColumn } from "../../../sdk"
import { TableSchema } from "./schema"
export const INTERNAL_TABLE_SOURCE_ID = "bb_internal"
@ -30,6 +30,5 @@ export interface Table extends Document {
export interface TableRequest extends Table {
_rename?: RenameColumn
_add?: AddColumn
created?: boolean
}

7
packages/types/src/documents/global/user.ts
View File

@ -22,6 +22,13 @@ export interface UserSSO {
providerType: SSOProviderType
oauth2?: OAuth2
thirdPartyProfile?: SSOProfileJson
profile?: {
displayName?: string
name?: {
givenName?: string
familyName?: string
}
}
}
export type SSOUser = User & UserSSO

8
packages/types/src/sdk/datasources.ts
View File

@ -14,6 +14,14 @@ export enum Operation {
DELETE_TABLE = "DELETE_TABLE",
}
export const RowOperations = [
Operation.CREATE,
Operation.READ,
Operation.UPDATE,
Operation.DELETE,
Operation.BULK_CREATE,
]
export enum SortDirection {
ASCENDING = "ASCENDING",
DESCENDING = "DESCENDING",

9
packages/types/src/sdk/search.ts
View File

@ -27,10 +27,13 @@ export interface SearchFilters {
[key: string]: string
}
[SearchFilterOperator.RANGE]?: {
[key: string]: {
[key: string]:
| {
high: number | string
low: number | string
}
| { high: number | string }
| { low: number | string }
}
[SearchFilterOperator.EQUAL]?: {
[key: string]: any
@ -77,10 +80,6 @@ export interface RenameColumn {
updated: string
}
export interface AddColumn {
name: string
}
export interface RelationshipsJson {
through?: string
from?: string

5
packages/worker/src/api/controllers/global/users.ts
View File

@ -116,7 +116,8 @@ const parseBooleanParam = (param: any) => {
export const adminUser = async (
ctx: Ctx<CreateAdminUserRequest, CreateAdminUserResponse>
) => {
const { email, password, tenantId, ssoId } = ctx.request.body
const { email, password, tenantId, ssoId, givenName, familyName } =
ctx.request.body
if (await platform.tenants.exists(tenantId)) {
ctx.throw(403, "Organisation already exists.")
@ -151,6 +152,8 @@ export const adminUser = async (
ssoId,
hashPassword,
requirePassword,
firstName: givenName,
lastName: familyName,
})
// events

2
packages/worker/src/api/routes/global/users.ts
View File

@ -16,6 +16,8 @@ function buildAdminInitValidation() {
password: OPTIONAL_STRING,
tenantId: Joi.string().required(),
ssoId: Joi.string(),
familyName: OPTIONAL_STRING,
givenName: OPTIONAL_STRING,
})
.required()
.unknown(false)

12
scripts/devDocker.sh Executable file
View File

@ -0,0 +1,12 @@
#!/bin/bash
# Check if the pro submodule is loaded
if [ ! -d "./packages/pro/src" ]; then
echo "[ERROR] Submodule is not loaded. This is only allowed with loaded submodules."
exit 1
fi
yarn build --scope @budibase/server --scope @budibase/worker
docker-compose -f hosting/docker-compose.build.yaml -f hosting/docker-compose.dev.yaml --env-file hosting/.env up --build --scale proxy-service=0