MirrorSave
/
budibase
mirror of https://github.com/Budibase/budibase.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Merge remote-tracking branch 'origin/master' into feature/app-list-actions

This commit is contained in:
Dean 2024-03-05 16:24:14 +00:00
parent 320b443ca4 0d49d705b6
commit 136eeefc26
39 changed files with 816 additions and 225 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
lerna.json
View File

@ -1,5 +1,5 @@
{
"version": "2.21.0",
"version": "2.21.2",
"npmClient": "yarn",
"packages": [
"packages/*",

57
packages/backend-core/src/security/roles.ts
View File

@ -84,16 +84,18 @@ export function getBuiltinRoles(): { [key: string]: RoleDoc } {
return cloneDeep(BUILTIN_ROLES)
}
export const BUILTIN_ROLE_ID_ARRAY = Object.values(BUILTIN_ROLES).map(
role => role._id
)
export function isBuiltin(role: string) {
return getBuiltinRole(role) !== undefined
}
export const BUILTIN_ROLE_NAME_ARRAY = Object.values(BUILTIN_ROLES).map(
role => role.name
)
export function isBuiltin(role?: string) {
return BUILTIN_ROLE_ID_ARRAY.some(builtin => role?.includes(builtin))
export function getBuiltinRole(roleId: string): Role | undefined {
const role = Object.values(BUILTIN_ROLES).find(role =>
roleId.includes(role._id)
)
if (!role) {
return undefined
}
return cloneDeep(role)
}
/**
@ -123,7 +125,7 @@ export function builtinRoleToNumber(id?: string) {
/**
* Converts any role to a number, but has to be async to get the roles from db.
*/
export async function roleToNumber(id?: string) {
export async function roleToNumber(id: string) {
if (isBuiltin(id)) {
return builtinRoleToNumber(id)
}
@ -131,7 +133,7 @@ export async function roleToNumber(id?: string) {
defaultPublic: true,
})) as RoleDoc[]
for (let role of hierarchy) {
if (isBuiltin(role?.inherits)) {
if (role?.inherits && isBuiltin(role.inherits)) {
return builtinRoleToNumber(role.inherits) + 1
}
}
@ -161,35 +163,28 @@ export function lowerBuiltinRoleID(roleId1?: string, roleId2?: string): string {
* @returns The role object, which may contain an "inherits" property.
*/
export async function getRole(
roleId?: string,
roleId: string,
opts?: { defaultPublic?: boolean }
): Promise<RoleDoc | undefined> {
if (!roleId) {
return undefined
}
let role: any = {}
): Promise<RoleDoc> {
// built in roles mostly come from the in-code implementation,
// but can be extended by a doc stored about them (e.g. permissions)
if (isBuiltin(roleId)) {
role = cloneDeep(
Object.values(BUILTIN_ROLES).find(role => role._id === roleId)
)
} else {
let role: RoleDoc | undefined = getBuiltinRole(roleId)
if (!role) {
// make sure has the prefix (if it has it then it won't be added)
roleId = prefixRoleID(roleId)
}
try {
const db = getAppDB()
const dbRole = await db.get(getDBRoleID(roleId))
role = Object.assign(role, dbRole)
const dbRole = await db.get<RoleDoc>(getDBRoleID(roleId))
role = Object.assign(role || {}, dbRole)
// finalise the ID
role._id = getExternalRoleID(role._id, role.version)
role._id = getExternalRoleID(role._id!, role.version)
} catch (err) {
if (!isBuiltin(roleId) && opts?.defaultPublic) {
return cloneDeep(BUILTIN_ROLES.PUBLIC)
}
// only throw an error if there is no role at all
if (Object.keys(role).length === 0) {
if (!role || Object.keys(role).length === 0) {
throw err
}
}
@ -200,7 +195,7 @@ export async function getRole(
* Simple function to get all the roles based on the top level user role ID.
*/
async function getAllUserRoles(
userRoleId?: string,
userRoleId: string,
opts?: { defaultPublic?: boolean }
): Promise<RoleDoc[]> {
// admins have access to all roles
@ -226,7 +221,7 @@ async function getAllUserRoles(
}
export async function getUserRoleIdHierarchy(
userRoleId?: string
userRoleId: string
): Promise<string[]> {
const roles = await getUserRoleHierarchy(userRoleId)
return roles.map(role => role._id!)
@ -241,7 +236,7 @@ export async function getUserRoleIdHierarchy(
* highest level of access and the last being the lowest level.
*/
export async function getUserRoleHierarchy(
userRoleId?: string,
userRoleId: string,
opts?: { defaultPublic?: boolean }
) {
// special case, if they don't have a role then they are a public user
@ -265,9 +260,9 @@ export function checkForRoleResourceArray(
return rolePerms
}
export async function getAllRoleIds(appId?: string) {
export async function getAllRoleIds(appId: string): Promise<string[]> {
const roles = await getAllRoles(appId)
return roles.map(role => role._id)
return roles.map(role => role._id!)
}
/**

6
packages/builder/src/stores/builder/tables.js
View File

@ -147,6 +147,12 @@ export function createTablesStore() {
if (indexes) {
draft.indexes = indexes
}
// Add object to indicate if column is being added
if (draft.schema[field.name] === undefined) {
draft._add = {
name: field.name,
}
}
draft.schema = {
...draft.schema,
[field.name]: cloneDeep(field),

27
packages/client/src/components/app/forms/RelationshipField.svelte
View File

@ -1,7 +1,7 @@
<script>
import { CoreSelect, CoreMultiselect } from "@budibase/bbui"
import { fetchData, Utils } from "@budibase/frontend-core"
import { getContext } from "svelte"
import { getContext, onMount } from "svelte"
import Field from "./Field.svelte"
import { FieldTypes } from "../../../constants"
@ -28,6 +28,7 @@
let tableDefinition
let searchTerm
let open
let initialValue
$: type =
datasourceType === "table" ? FieldTypes.LINK : FieldTypes.BB_REFERENCE
@ -109,7 +110,11 @@
}
$: forceFetchRows(filter)
$: debouncedFetchRows(searchTerm, primaryDisplay, defaultValue)
$: debouncedFetchRows(
searchTerm,
primaryDisplay,
initialValue || defaultValue
)
const forceFetchRows = async () => {
// if the filter has changed, then we need to reset the options, clear the selection, and re-fetch
@ -127,9 +132,13 @@
if (allRowsFetched || !primaryDisplay) {
return
}
if (defaultVal && !optionsObj[defaultVal]) {
// must be an array
if (defaultVal && !Array.isArray(defaultVal)) {
defaultVal = defaultVal.split(",")
}
if (defaultVal && defaultVal.some(val => !optionsObj[val])) {
await fetch.update({
query: { equal: { _id: defaultVal } },
query: { oneOf: { _id: defaultVal } },
})
}
@ -202,6 +211,16 @@
fetch.nextPage()
}
}
onMount(() => {
// if the form is in 'Update' mode, then we need to fetch the matching row so that the value is correctly set
if (fieldState?.value) {
initialValue =
fieldSchema?.relationshipType !== "one-to-many"
? flatten(fieldState?.value) ?? []
: flatten(fieldState?.value)?.[0]
}
})
</script>
<Field

6
packages/server/scripts/integrations/mysql/init.sql
View File

@ -10,6 +10,11 @@ CREATE TABLE Persons (
City varchar(255),
PRIMARY KEY (PersonID)
);
CREATE TABLE Person (
PersonID int NOT NULL AUTO_INCREMENT,
Name varchar(255),
PRIMARY KEY (PersonID)
);
CREATE TABLE Tasks (
TaskID int NOT NULL AUTO_INCREMENT,
PersonID INT,
@ -27,6 +32,7 @@ CREATE TABLE Products (
);
INSERT INTO Persons (FirstName, LastName, Age, Address, City, CreatedAt) VALUES ('Mike', 'Hughes', 28.2, '123 Fake Street', 'Belfast', '2021-01-19 03:14:07');
INSERT INTO Persons (FirstName, LastName, Age, Address, City, CreatedAt) VALUES ('Dave', 'Johnson', 29, '124 Fake Street', 'Belfast', '2022-04-01 00:11:11');
INSERT INTO Person (Name) VALUES ('Elf');
INSERT INTO Tasks (PersonID, TaskName, CreatedAt) VALUES (1, 'assembling', '2020-01-01');
INSERT INTO Tasks (PersonID, TaskName, CreatedAt) VALUES (2, 'processing', '2019-12-31');
INSERT INTO Products (name, updated) VALUES ('Meat', '11:00:22'), ('Fruit', '10:00:00');

61
packages/server/src/api/controllers/role.ts
View File

@ -7,8 +7,14 @@ import {
} from "@budibase/backend-core"
import { getUserMetadataParams, InternalTables } from "../../db/utils"
import {
AccessibleRolesResponse,
Database,
DestroyRoleResponse,
FetchRolesResponse,
FindRoleResponse,
Role,
SaveRoleRequest,
SaveRoleResponse,
UserCtx,
UserMetadata,
UserRoles,
@ -25,43 +31,36 @@ async function updateRolesOnUserTable(
db: Database,
roleId: string,
updateOption: string,
roleVersion: string | undefined
roleVersion?: string
) {
const table = await sdk.tables.getTable(InternalTables.USER_METADATA)
const schema = table.schema
const constraints = table.schema.roleId?.constraints
if (!constraints) {
return
}
const updatedRoleId =
roleVersion === roles.RoleIDVersion.NAME
? roles.getExternalRoleID(roleId, roleVersion)
: roleId
const indexOfRoleId = constraints.inclusion!.indexOf(updatedRoleId)
const remove = updateOption === UpdateRolesOptions.REMOVED
let updated = false
for (let prop of Object.keys(schema)) {
if (prop === "roleId") {
updated = true
const constraints = schema[prop].constraints!
const updatedRoleId =
roleVersion === roles.RoleIDVersion.NAME
? roles.getExternalRoleID(roleId, roleVersion)
: roleId
const indexOfRoleId = constraints.inclusion!.indexOf(updatedRoleId)
if (remove && indexOfRoleId !== -1) {
constraints.inclusion!.splice(indexOfRoleId, 1)
} else if (!remove && indexOfRoleId === -1) {
constraints.inclusion!.push(updatedRoleId)
}
break
}
}
if (updated) {
await db.put(table)
if (remove && indexOfRoleId !== -1) {
constraints.inclusion!.splice(indexOfRoleId, 1)
} else if (!remove && indexOfRoleId === -1) {
constraints.inclusion!.push(updatedRoleId)
}
await db.put(table)
}
export async function fetch(ctx: UserCtx) {
export async function fetch(ctx: UserCtx<void, FetchRolesResponse>) {
ctx.body = await roles.getAllRoles()
}
export async function find(ctx: UserCtx) {
export async function find(ctx: UserCtx<void, FindRoleResponse>) {
ctx.body = await roles.getRole(ctx.params.roleId)
}
export async function save(ctx: UserCtx) {
export async function save(ctx: UserCtx<SaveRoleRequest, SaveRoleResponse>) {
const db = context.getAppDB()
let { _id, name, inherits, permissionId, version } = ctx.request.body
let isCreate = false
@ -109,9 +108,9 @@ export async function save(ctx: UserCtx) {
ctx.body = role
}
export async function destroy(ctx: UserCtx) {
export async function destroy(ctx: UserCtx<void, DestroyRoleResponse>) {
const db = context.getAppDB()
let roleId = ctx.params.roleId
let roleId = ctx.params.roleId as string
if (roles.isBuiltin(roleId)) {
ctx.throw(400, "Cannot delete builtin role.")
} else {
@ -144,14 +143,18 @@ export async function destroy(ctx: UserCtx) {
ctx.status = 200
}
export async function accessible(ctx: UserCtx) {
export async function accessible(ctx: UserCtx<void, AccessibleRolesResponse>) {
let roleId = ctx.user?.roleId
if (!roleId) {
roleId = roles.BUILTIN_ROLE_IDS.PUBLIC
}
if (ctx.user && sharedSdk.users.isAdminOrBuilder(ctx.user)) {
const appId = context.getAppId()
ctx.body = await roles.getAllRoleIds(appId)
if (!appId) {
ctx.body = []
} else {
ctx.body = await roles.getAllRoleIds(appId)
}
} else {
ctx.body = await roles.getUserRoleIdHierarchy(roleId!)
}

2
packages/server/src/api/controllers/routing.ts
View File

@ -63,7 +63,7 @@ export async function fetch(ctx: UserCtx) {
export async function clientFetch(ctx: UserCtx) {
const routing = await getRoutingStructure()
let roleId = ctx.user?.role?._id
const roleIds = await roles.getUserRoleIdHierarchy(roleId)
const roleIds = roleId ? await roles.getUserRoleIdHierarchy(roleId) : []
for (let topLevel of Object.values(routing.routes) as any) {
for (let subpathKey of Object.keys(topLevel.subpaths)) {
let found = false

98
packages/server/src/api/controllers/row/alias.ts
View File

@ -62,7 +62,11 @@ export default class AliasTables {
if (idx === -1 || idx > 1) {
return
}
return Math.abs(tableName.length - name.length) <= 2
// this might look a bit mad, but the idea is if the field is wrapped, say in "", `` or []
// then the idx of the table name will be 1, and we should allow for it ending in a closing
// character - otherwise it should be the full length if the index is zero
const allowedCharacterDiff = idx * 2
return Math.abs(tableName.length - name.length) <= allowedCharacterDiff
})
if (foundTableName) {
const aliasedTableName = tableName.replace(
@ -107,57 +111,55 @@ export default class AliasTables {
}
async queryWithAliasing(json: QueryJson): DatasourcePlusQueryResponse {
json = cloneDeep(json)
const aliasTable = (table: Table) => ({
...table,
name: this.getAlias(table.name),
})
// run through the query json to update anywhere a table may be used
if (json.resource?.fields) {
json.resource.fields = json.resource.fields.map(field =>
this.aliasField(field)
)
}
if (json.filters) {
for (let [filterKey, filter] of Object.entries(json.filters)) {
if (typeof filter !== "object") {
continue
}
const aliasedFilters: typeof filter = {}
for (let key of Object.keys(filter)) {
aliasedFilters[this.aliasField(key)] = filter[key]
}
json.filters[filterKey as keyof SearchFilters] = aliasedFilters
const fieldLength = json.resource?.fields?.length
const aliasingEnabled = fieldLength && fieldLength > 0
if (aliasingEnabled) {
json = cloneDeep(json)
// run through the query json to update anywhere a table may be used
if (json.resource?.fields) {
json.resource.fields = json.resource.fields.map(field =>
this.aliasField(field)
)
}
}
if (json.relationships) {
json.relationships = json.relationships.map(relationship => ({
...relationship,
aliases: this.aliasMap([
relationship.through,
relationship.tableName,
json.endpoint.entityId,
]),
}))
}
if (json.meta?.table) {
json.meta.table = aliasTable(json.meta.table)
}
if (json.meta?.tables) {
const aliasedTables: Record<string, Table> = {}
for (let [tableName, table] of Object.entries(json.meta.tables)) {
aliasedTables[this.getAlias(tableName)] = aliasTable(table)
if (json.filters) {
for (let [filterKey, filter] of Object.entries(json.filters)) {
if (typeof filter !== "object") {
continue
}
const aliasedFilters: typeof filter = {}
for (let key of Object.keys(filter)) {
aliasedFilters[this.aliasField(key)] = filter[key]
}
json.filters[filterKey as keyof SearchFilters] = aliasedFilters
}
}
json.meta.tables = aliasedTables
if (json.meta?.table) {
this.getAlias(json.meta.table.name)
}
if (json.meta?.tables) {
Object.keys(json.meta.tables).forEach(tableName =>
this.getAlias(tableName)
)
}
if (json.relationships) {
json.relationships = json.relationships.map(relationship => ({
...relationship,
aliases: this.aliasMap([
relationship.through,
relationship.tableName,
json.endpoint.entityId,
]),
}))
}
// invert and return
const invertedTableAliases: Record<string, string> = {}
for (let [key, value] of Object.entries(this.tableAliases)) {
invertedTableAliases[value] = key
}
json.tableAliases = invertedTableAliases
}
// invert and return
const invertedTableAliases: Record<string, string> = {}
for (let [key, value] of Object.entries(this.tableAliases)) {
invertedTableAliases[value] = key
}
json.tableAliases = invertedTableAliases
const response = await getDatasourceAndQuery(json)
if (Array.isArray(response)) {
if (Array.isArray(response) && aliasingEnabled) {
return this.reverse(response)
} else {
return response

10
packages/server/src/api/controllers/table/external.ts
View File

@ -6,6 +6,7 @@ import {
BulkImportRequest,
BulkImportResponse,
Operation,
RenameColumn,
SaveTableRequest,
SaveTableResponse,
Table,
@ -25,9 +26,12 @@ function getDatasourceId(table: Table) {
return breakExternalTableId(table._id).datasourceId
}
export async function save(ctx: UserCtx<SaveTableRequest, SaveTableResponse>) {
export async function save(
ctx: UserCtx<SaveTableRequest, SaveTableResponse>,
renaming?: RenameColumn
) {
const inputs = ctx.request.body
const renaming = inputs?._rename
const adding = inputs?._add
// can't do this right now
delete inputs.rows
const tableId = ctx.request.body._id
@ -40,7 +44,7 @@ export async function save(ctx: UserCtx<SaveTableRequest, SaveTableResponse>) {
const { datasource, table } = await sdk.tables.external.save(
datasourceId!,
inputs,
{ tableId, renaming }
{ tableId, renaming, adding }
)
builderSocket?.emitDatasourceUpdate(ctx, datasource)
return table

9
packages/server/src/api/controllers/table/index.ts
View File

@ -74,8 +74,15 @@ export async function save(ctx: UserCtx<SaveTableRequest, SaveTableResponse>) {
const appId = ctx.appId
const table = ctx.request.body
const isImport = table.rows
const renaming = ctx.request.body._rename
let savedTable = await pickApi({ table }).save(ctx)
const api = pickApi({ table })
// do not pass _rename or _add if saving to CouchDB
if (api === internal) {
delete ctx.request.body._add
delete ctx.request.body._rename
}
let savedTable = await api.save(ctx, renaming)
if (!table._id) {
await events.table.created(savedTable)
savedTable = sdk.tables.enrichViewSchemas(savedTable)

14
packages/server/src/api/controllers/table/internal.ts
View File

@ -12,11 +12,12 @@ import {
} from "@budibase/types"
import sdk from "../../../sdk"
export async function save(ctx: UserCtx<SaveTableRequest, SaveTableResponse>) {
export async function save(
ctx: UserCtx<SaveTableRequest, SaveTableResponse>,
renaming?: RenameColumn
) {
const { rows, ...rest } = ctx.request.body
let tableToSave: Table & {
_rename?: RenameColumn
} = {
let tableToSave: Table = {
_id: generateTableID(),
...rest,
// Ensure these fields are populated, even if not sent in the request
@ -28,15 +29,12 @@ export async function save(ctx: UserCtx<SaveTableRequest, SaveTableResponse>) {
tableToSave.views = {}
}
const renaming = tableToSave._rename
delete tableToSave._rename
try {
const { table } = await sdk.tables.internal.save(tableToSave, {
user: ctx.user,
rowsToImport: rows,
tableId: ctx.request.body._id,
renaming: renaming,
renaming,
})
return table

4
packages/server/src/api/routes/tests/apikeys.spec.js
View File

@ -13,7 +13,7 @@ describe("/api/keys", () => {
describe("fetch", () => {
it("should allow fetching", async () => {
await setup.switchToSelfHosted(async () => {
await config.withEnv({ SELF_HOSTED: "true" }, async () => {
const res = await request
.get(`/api/keys`)
.set(config.defaultHeaders())
@ -34,7 +34,7 @@ describe("/api/keys", () => {
describe("update", () => {
it("should allow updating a value", async () => {
await setup.switchToSelfHosted(async () => {
await config.withEnv({ SELF_HOSTED: "true" }, async () => {
const res = await request
.put(`/api/keys/TEST`)
.send({

14
packages/server/src/api/routes/tests/application.spec.ts
View File

@ -318,4 +318,18 @@ describe("/applications", () => {
expect(devLogs.data.length).toBe(0)
})
})
describe("permissions", () => {
it("should only return apps a user has access to", async () => {
const user = await config.createUser({
builder: { global: false },
admin: { global: false },
})
await config.withUser(user, async () => {
const apps = await config.api.application.fetch()
expect(apps).toHaveLength(0)
})
})
})
})

2
packages/server/src/api/routes/tests/queries/query.seq.spec.ts
View File

@ -157,7 +157,7 @@ describe("/queries", () => {
})
it("should find a query in cloud", async () => {
await setup.switchToSelfHosted(async () => {
await config.withEnv({ SELF_HOSTED: "true" }, async () => {
const query = await config.createQuery()
const res = await request
.get(`/api/queries/${query._id}`)

3
packages/server/src/api/routes/tests/row.spec.ts
View File

@ -882,8 +882,7 @@ describe.each([
],
tableId: table._id,
})
// the environment needs configured for this
await setup.switchToSelfHosted(async () => {
await config.withEnv({ SELF_HOSTED: "true" }, async () => {
return context.doInAppContext(config.getAppId(), async () => {
const enriched = await outputProcessing(table, [row])
expect((enriched as Row[])[0].attachment[0].url).toBe(

30
packages/server/src/api/routes/tests/table.spec.ts
View File

@ -26,6 +26,7 @@ import { TableToBuild } from "../../../tests/utilities/TestConfiguration"
tk.freeze(mocks.date.MOCK_DATE)
const { basicTable } = setup.structures
const ISO_REGEX_PATTERN = /^\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}\.\d{3}Z$/
describe("/tables", () => {
let request = setup.getRequest()
@ -285,6 +286,35 @@ describe("/tables", () => {
expect(res.body.schema.roleId).toBeDefined()
})
})
it("should add a new column for an internal DB table", async () => {
const saveTableRequest: SaveTableRequest = {
_add: {
name: "NEW_COLUMN",
},
...basicTable(),
}
const response = await request
.post(`/api/tables`)
.send(saveTableRequest)
.set(config.defaultHeaders())
.expect("Content-Type", /json/)
.expect(200)
const expectedResponse = {
...saveTableRequest,
_rev: expect.stringMatching(/^\d-.+/),
_id: expect.stringMatching(/^ta_.+/),
createdAt: expect.stringMatching(ISO_REGEX_PATTERN),
updatedAt: expect.stringMatching(ISO_REGEX_PATTERN),
views: {},
}
delete expectedResponse._add
expect(response.status).toBe(200)
expect(response.body).toEqual(expectedResponse)
})
})
describe("import", () => {

25
packages/server/src/api/routes/tests/utilities/index.ts
View File

@ -1,5 +1,4 @@
import TestConfig from "../../../../tests/utilities/TestConfiguration"
import env from "../../../../environment"
import TestConfiguration from "../../../../tests/utilities/TestConfiguration"
import supertest from "supertest"
export * as structures from "../../../../tests/utilities/structures"
@ -47,10 +46,10 @@ export function delay(ms: number) {
}
let request: supertest.SuperTest<supertest.Test> | undefined | null,
config: TestConfig | null
config: TestConfiguration | null
export function beforeAll() {
config = new TestConfig()
config = new TestConfiguration()
request = config.getRequest()
}
@ -77,21 +76,3 @@ export function getConfig() {
}
return config!
}
export async function switchToSelfHosted(func: any) {
// self hosted stops any attempts to Dynamo
env._set("NODE_ENV", "production")
env._set("SELF_HOSTED", true)
let error
try {
await func()
} catch (err) {
error = err
}
env._set("NODE_ENV", "jest")
env._set("SELF_HOSTED", false)
// don't throw error until after reset
if (error) {
throw error
}
}

363
packages/server/src/integration-test/mysql.spec.ts Normal file
View File

@ -0,0 +1,363 @@
import fetch from "node-fetch"
import {
generateMakeRequest,
MakeRequestResponse,
} from "../api/routes/public/tests/utils"
import { v4 as uuidv4 } from "uuid"
import * as setup from "../api/routes/tests/utilities"
import {
Datasource,
FieldType,
Table,
TableRequest,
TableSourceType,
} from "@budibase/types"
import _ from "lodash"
import { databaseTestProviders } from "../integrations/tests/utils"
import mysql from "mysql2/promise"
import { builderSocket } from "../websockets"
// @ts-ignore
fetch.mockSearch()
const config = setup.getConfig()!
jest.unmock("mysql2/promise")
jest.mock("../websockets", () => ({
clientAppSocket: jest.fn(),
gridAppSocket: jest.fn(),
initialise: jest.fn(),
builderSocket: {
emitTableUpdate: jest.fn(),
emitTableDeletion: jest.fn(),
emitDatasourceUpdate: jest.fn(),
emitDatasourceDeletion: jest.fn(),
emitScreenUpdate: jest.fn(),
emitAppMetadataUpdate: jest.fn(),
emitAppPublish: jest.fn(),
},
}))
describe("mysql integrations", () => {
let makeRequest: MakeRequestResponse,
mysqlDatasource: Datasource,
primaryMySqlTable: Table
beforeAll(async () => {
await config.init()
const apiKey = await config.generateApiKey()
makeRequest = generateMakeRequest(apiKey, true)
mysqlDatasource = await config.api.datasource.create(
await databaseTestProviders.mysql.datasource()
)
})
afterAll(async () => {
await databaseTestProviders.mysql.stop()
})
beforeEach(async () => {
primaryMySqlTable = await config.createTable({
name: uuidv4(),
type: "table",
primary: ["id"],
schema: {
id: {
name: "id",
type: FieldType.AUTO,
autocolumn: true,
},
name: {
name: "name",
type: FieldType.STRING,
},
description: {
name: "description",
type: FieldType.STRING,
},
value: {
name: "value",
type: FieldType.NUMBER,
},
},
sourceId: mysqlDatasource._id,
sourceType: TableSourceType.EXTERNAL,
})
})
afterAll(config.end)
it("validate table schema", async () => {
const res = await makeRequest(
"get",
`/api/datasources/${mysqlDatasource._id}`
)
expect(res.status).toBe(200)
expect(res.body).toEqual({
config: {
database: "mysql",
host: mysqlDatasource.config!.host,
password: "--secret-value--",
port: mysqlDatasource.config!.port,
user: "root",
},
plus: true,
source: "MYSQL",
type: "datasource_plus",
_id: expect.any(String),
_rev: expect.any(String),
createdAt: expect.any(String),
updatedAt: expect.any(String),
entities: expect.any(Object),
})
})
describe("POST /api/datasources/verify", () => {
it("should be able to verify the connection", async () => {
await config.api.datasource.verify(
{
datasource: await databaseTestProviders.mysql.datasource(),
},
{
body: {
connected: true,
},
}
)
})
it("should state an invalid datasource cannot connect", async () => {
const dbConfig = await databaseTestProviders.mysql.datasource()
await config.api.datasource.verify(
{
datasource: {
...dbConfig,
config: {
...dbConfig.config,
password: "wrongpassword",
},
},
},
{
body: {
connected: false,
error:
"Access denied for the specified user. User does not have the necessary privileges or the provided credentials are incorrect. Please verify the credentials, and ensure that the user has appropriate permissions.",
},
}
)
})
})
describe("POST /api/datasources/info", () => {
it("should fetch information about mysql datasource", async () => {
const primaryName = primaryMySqlTable.name
const response = await makeRequest("post", "/api/datasources/info", {
datasource: mysqlDatasource,
})
expect(response.status).toBe(200)
expect(response.body.tableNames).toBeDefined()
expect(response.body.tableNames.indexOf(primaryName)).not.toBe(-1)
})
})
describe("Integration compatibility with mysql search_path", () => {
let client: mysql.Connection, pathDatasource: Datasource
const database = "test1"
const database2 = "test-2"
beforeAll(async () => {
const dsConfig = await databaseTestProviders.mysql.datasource()
const dbConfig = dsConfig.config!
client = await mysql.createConnection(dbConfig)
await client.query(`CREATE DATABASE \`${database}\`;`)
await client.query(`CREATE DATABASE \`${database2}\`;`)
const pathConfig: any = {
...dsConfig,
config: {
...dbConfig,
database,
},
}
pathDatasource = await config.api.datasource.create(pathConfig)
})
afterAll(async () => {
await client.query(`DROP DATABASE \`${database}\`;`)
await client.query(`DROP DATABASE \`${database2}\`;`)
await client.end()
})
it("discovers tables from any schema in search path", async () => {
await client.query(
`CREATE TABLE \`${database}\`.table1 (id1 SERIAL PRIMARY KEY);`
)
const response = await makeRequest("post", "/api/datasources/info", {
datasource: pathDatasource,
})
expect(response.status).toBe(200)
expect(response.body.tableNames).toBeDefined()
expect(response.body.tableNames).toEqual(
expect.arrayContaining(["table1"])
)
})
it("does not mix columns from different tables", async () => {
const repeated_table_name = "table_same_name"
await client.query(
`CREATE TABLE \`${database}\`.${repeated_table_name} (id SERIAL PRIMARY KEY, val1 TEXT);`
)
await client.query(
`CREATE TABLE \`${database2}\`.${repeated_table_name} (id2 SERIAL PRIMARY KEY, val2 TEXT);`
)
const response = await makeRequest(
"post",
`/api/datasources/${pathDatasource._id}/schema`,
{
tablesFilter: [repeated_table_name],
}
)
expect(response.status).toBe(200)
expect(
response.body.datasource.entities[repeated_table_name].schema
).toBeDefined()
const schema =
response.body.datasource.entities[repeated_table_name].schema
expect(Object.keys(schema).sort()).toEqual(["id", "val1"])
})
})
describe("POST /api/tables/", () => {
let client: mysql.Connection
const emitDatasourceUpdateMock = jest.fn()
beforeEach(async () => {
client = await mysql.createConnection(
(
await databaseTestProviders.mysql.datasource()
).config!
)
mysqlDatasource = await config.api.datasource.create(
await databaseTestProviders.mysql.datasource()
)
})
afterEach(async () => {
await client.end()
})
it("will emit the datasource entity schema with externalType to the front-end when adding a new column", async () => {
const addColumnToTable: TableRequest = {
type: "table",
sourceType: TableSourceType.EXTERNAL,
name: "table",
sourceId: mysqlDatasource._id!,
primary: ["id"],
schema: {
id: {
type: FieldType.AUTO,
name: "id",
autocolumn: true,
},
new_column: {
type: FieldType.NUMBER,
name: "new_column",
},
},
_add: {
name: "new_column",
},
}
jest
.spyOn(builderSocket!, "emitDatasourceUpdate")
.mockImplementation(emitDatasourceUpdateMock)
await makeRequest("post", "/api/tables/", addColumnToTable)
const expectedTable: TableRequest = {
...addColumnToTable,
schema: {
id: {
type: FieldType.NUMBER,
name: "id",
autocolumn: true,
constraints: {
presence: false,
},
externalType: "int unsigned",
},
new_column: {
type: FieldType.NUMBER,
name: "new_column",
autocolumn: false,
constraints: {
presence: false,
},
externalType: "float(8,2)",
},
},
created: true,
_id: `${mysqlDatasource._id}__table`,
}
delete expectedTable._add
expect(emitDatasourceUpdateMock).toBeCalledTimes(1)
const emittedDatasource: Datasource =
emitDatasourceUpdateMock.mock.calls[0][1]
expect(emittedDatasource.entities!["table"]).toEqual(expectedTable)
})
it("will rename a column", async () => {
await makeRequest("post", "/api/tables/", primaryMySqlTable)
let renameColumnOnTable: TableRequest = {
...primaryMySqlTable,
schema: {
id: {
name: "id",
type: FieldType.AUTO,
autocolumn: true,
externalType: "unsigned integer",
},
name: {
name: "name",
type: FieldType.STRING,
externalType: "text",
},
description: {
name: "description",
type: FieldType.STRING,
externalType: "text",
},
age: {
name: "age",
type: FieldType.NUMBER,
externalType: "float(8,2)",
},
},
}
const response = await makeRequest(
"post",
"/api/tables/",
renameColumnOnTable
)
mysqlDatasource = (
await makeRequest(
"post",
`/api/datasources/${mysqlDatasource._id}/schema`
)
).body.datasource
expect(response.status).toEqual(200)
expect(
Object.keys(mysqlDatasource.entities![primaryMySqlTable.name].schema)
).toEqual(["id", "name", "description", "age"])
})
})
})

15
packages/server/src/integrations/base/sql.ts
View File

@ -12,6 +12,8 @@ import {
} from "@budibase/types"
import environment from "../../environment"
type QueryFunction = (query: Knex.SqlNative, operation: Operation) => any
const envLimit = environment.SQL_MAX_ROWS
? parseInt(environment.SQL_MAX_ROWS)
: null
@ -322,15 +324,18 @@ class InternalBuilder {
addSorting(query: Knex.QueryBuilder, json: QueryJson): Knex.QueryBuilder {
let { sort, paginate } = json
const table = json.meta?.table
const aliases = json.tableAliases
const aliased =
table?.name && aliases?.[table.name] ? aliases[table.name] : table?.name
if (sort && Object.keys(sort || {}).length > 0) {
for (let [key, value] of Object.entries(sort)) {
const direction =
value.direction === SortDirection.ASCENDING ? "asc" : "desc"
query = query.orderBy(`${table?.name}.${key}`, direction)
query = query.orderBy(`${aliased}.${key}`, direction)
}
} else if (this.client === SqlClient.MS_SQL && paginate?.limit) {
// @ts-ignore
query = query.orderBy(`${table?.name}.${table?.primary[0]}`)
query = query.orderBy(`${aliased}.${table?.primary[0]}`)
}
return query
}
@ -605,7 +610,7 @@ class SqlQueryBuilder extends SqlTableQueryBuilder {
return query.toSQL().toNative()
}
async getReturningRow(queryFn: Function, json: QueryJson) {
async getReturningRow(queryFn: QueryFunction, json: QueryJson) {
if (!json.extra || !json.extra.idFilter) {
return {}
}
@ -617,7 +622,7 @@ class SqlQueryBuilder extends SqlTableQueryBuilder {
resource: {
fields: [],
},
filters: json.extra.idFilter,
filters: json.extra?.idFilter,
paginate: {
limit: 1,
},
@ -646,7 +651,7 @@ class SqlQueryBuilder extends SqlTableQueryBuilder {
// this function recreates the returning functionality of postgres
async queryWithReturning(
json: QueryJson,
queryFn: Function,
queryFn: QueryFunction,
processFn: Function = (result: any) => result
) {
const sqlClient = this.getSqlClient()

23
packages/server/src/integrations/tests/sqlAlias.spec.ts
View File

@ -4,6 +4,7 @@ import Sql from "../base/sql"
import { SqlClient } from "../utils"
import AliasTables from "../../api/controllers/row/alias"
import { generator } from "@budibase/backend-core/tests"
import { Knex } from "knex"
function multiline(sql: string) {
return sql.replace(/\n/g, "").replace(/ +/g, " ")
@ -160,6 +161,28 @@ describe("Captures of real examples", () => {
})
})
describe("returning (everything bar Postgres)", () => {
it("should be able to handle row returning", () => {
const queryJson = getJson("createSimple.json")
const SQL = new Sql(SqlClient.MS_SQL, limit)
let query = SQL._query(queryJson, { disableReturning: true })
expect(query).toEqual({
sql: "insert into [people] ([age], [name]) values (@p0, @p1)",
bindings: [22, "Test"],
})
// now check returning
let returningQuery: Knex.SqlNative = { sql: "", bindings: [] }
SQL.getReturningRow((input: Knex.SqlNative) => {
returningQuery = input
}, queryJson)
expect(returningQuery).toEqual({
sql: "select * from (select top (@p0) * from [people] where [people].[name] = @p1 and [people].[age] = @p2 order by [people].[name] asc) as [people]",
bindings: [1, "Test", 22],
})
})
})
describe("check max character aliasing", () => {
it("should handle over 'z' max character alias", () => {
const tableNames = []

2
packages/server/src/integrations/tests/sqlQueryJson/basicFetchWithRelationships.json
View File

@ -68,7 +68,7 @@
"primary": [
"personid"
],
"name": "a",
"name": "persons",
"schema": {
"year": {
"type": "number",

64
packages/server/src/integrations/tests/sqlQueryJson/createSimple.json Normal file
View File

@ -0,0 +1,64 @@
{
"endpoint": {
"datasourceId": "datasource_plus_0ed5835e5552496285df546030f7c4ae",
"entityId": "people",
"operation": "CREATE"
},
"resource": {
"fields": [
"a.name",
"a.age"
]
},
"filters": {},
"relationships": [],
"body": {
"name": "Test",
"age": 22
},
"extra": {
"idFilter": {
"equal": {
"name": "Test",
"age": 22
}
}
},
"meta": {
"table": {
"_id": "datasource_plus_0ed5835e5552496285df546030f7c4ae__people",
"type": "table",
"sourceId": "datasource_plus_0ed5835e5552496285df546030f7c4ae",
"sourceType": "external",
"primary": [
"name",
"age"
],
"name": "people",
"schema": {
"name": {
"type": "string",
"externalType": "varchar",
"autocolumn": false,
"name": "name",
"constraints": {
"presence": true
}
},
"age": {
"type": "number",
"externalType": "int",
"autocolumn": false,
"name": "age",
"constraints": {
"presence": false
}
}
},
"primaryDisplay": "name"
}
},
"tableAliases": {
"people": "a"
}
}

2
packages/server/src/integrations/tests/sqlQueryJson/createWithRelationships.json
View File

@ -58,7 +58,7 @@
"primary": [
"personid"
],
"name": "a",
"name": "persons",
"schema": {
"year": {
"type": "number",

2
packages/server/src/integrations/tests/sqlQueryJson/deleteSimple.json
View File

@ -34,7 +34,7 @@
"keypartone",
"keyparttwo"
],
"name": "a",
"name": "compositetable",
"schema": {
"keyparttwo": {
"type": "string",

2
packages/server/src/integrations/tests/sqlQueryJson/enrichRelationship.json
View File

@ -49,7 +49,7 @@
"primary": [
"taskid"
],
"name": "a",
"name": "tasks",
"schema": {
"executorid": {
"type": "number",

2
packages/server/src/integrations/tests/sqlQueryJson/fetchManyToMany.json
View File

@ -63,7 +63,7 @@
"primary": [
"productid"
],
"name": "a",
"name": "products",
"schema": {
"productname": {
"type": "string",

2
packages/server/src/integrations/tests/sqlQueryJson/filterByRelationship.json
View File

@ -53,7 +53,7 @@
"primary": [
"productid"
],
"name": "a",
"name": "products",
"schema": {
"productname": {
"type": "string",

2
packages/server/src/integrations/tests/sqlQueryJson/manyRelationshipFilters.json
View File

@ -109,7 +109,7 @@
"primary": [
"taskid"
],
"name": "a",
"name": "tasks",
"schema": {
"executorid": {
"type": "number",

2
packages/server/src/integrations/tests/sqlQueryJson/updateRelationship.json
View File

@ -66,7 +66,7 @@
"primary": [
"personid"
],
"name": "a",
"name": "persons",
"schema": {
"year": {
"type": "number",

2
packages/server/src/integrations/tests/sqlQueryJson/updateSimple.json
View File

@ -66,7 +66,7 @@
"primary": [
"personid"
],
"name": "a",
"name": "persons",
"schema": {
"year": {
"type": "number",

18
packages/server/src/sdk/app/rows/search/external.ts
View File

@ -11,7 +11,10 @@ import {
import * as exporters from "../../../../api/controllers/view/exporters"
import sdk from "../../../../sdk"
import { handleRequest } from "../../../../api/controllers/row/external"
import { breakExternalTableId } from "../../../../integrations/utils"
import {
breakExternalTableId,
breakRowIdField,
} from "../../../../integrations/utils"
import { cleanExportRows } from "../utils"
import { utils } from "@budibase/shared-core"
import { ExportRowsParams, ExportRowsResult } from "../search"
@ -52,6 +55,15 @@ export async function search(options: SearchParams) {
}
}
// Make sure oneOf _id queries decode the Row IDs
if (query?.oneOf?._id) {
const rowIds = query.oneOf._id
query.oneOf._id = rowIds.map((row: string) => {
const ids = breakRowIdField(row)
return ids[0]
})
}
try {
const table = await sdk.tables.getTable(tableId)
options = searchInputMapping(table, options)
@ -119,9 +131,7 @@ export async function exportRows(
requestQuery = {
oneOf: {
_id: rowIds.map((row: string) => {
const ids = JSON.parse(
decodeURI(row).replace(/'/g, `"`).replace(/%2C/g, ",")
)
const ids = breakRowIdField(row)
if (ids.length > 1) {
throw new HTTPError(
"Export data does not support composite keys.",

115
packages/server/src/sdk/app/rows/search/tests/external.spec.ts
View File

@ -21,10 +21,11 @@ jest.unmock("mysql2/promise")
jest.setTimeout(30000)
describe.skip("external", () => {
describe("external search", () => {
const config = new TestConfiguration()
let externalDatasource: Datasource, tableData: Table
const rows: Row[] = []
beforeAll(async () => {
const container = await new GenericContainer("mysql")
@ -89,67 +90,81 @@ describe.skip("external", () => {
},
},
}
const table = await config.createExternalTable({
...tableData,
sourceId: externalDatasource._id,
})
for (let i = 0; i < 10; i++) {
rows.push(
await config.createRow({
tableId: table._id,
name: generator.first(),
surname: generator.last(),
age: generator.age(),
address: generator.address(),
})
)
}
})
describe("search", () => {
const rows: Row[] = []
beforeAll(async () => {
const table = await config.createExternalTable({
...tableData,
sourceId: externalDatasource._id,
})
for (let i = 0; i < 10; i++) {
rows.push(
await config.createRow({
tableId: table._id,
name: generator.first(),
surname: generator.last(),
age: generator.age(),
address: generator.address(),
})
)
it("default search returns all the data", async () => {
await config.doInContext(config.appId, async () => {
const tableId = config.table!._id!
const searchParams: SearchParams = {
tableId,
query: {},
}
const result = await search(searchParams)
expect(result.rows).toHaveLength(10)
expect(result.rows).toEqual(
expect.arrayContaining(rows.map(r => expect.objectContaining(r)))
)
})
})
it("default search returns all the data", async () => {
await config.doInContext(config.appId, async () => {
const tableId = config.table!._id!
it("querying by fields will always return data attribute columns", async () => {
await config.doInContext(config.appId, async () => {
const tableId = config.table!._id!
const searchParams: SearchParams = {
tableId,
query: {},
}
const result = await search(searchParams)
const searchParams: SearchParams = {
tableId,
query: {},
fields: ["name", "age"],
}
const result = await search(searchParams)
expect(result.rows).toHaveLength(10)
expect(result.rows).toEqual(
expect.arrayContaining(rows.map(r => expect.objectContaining(r)))
expect(result.rows).toHaveLength(10)
expect(result.rows).toEqual(
expect.arrayContaining(
rows.map(r => ({
...expectAnyExternalColsAttributes,
name: r.name,
age: r.age,
}))
)
})
)
})
})
it("querying by fields will always return data attribute columns", async () => {
await config.doInContext(config.appId, async () => {
const tableId = config.table!._id!
it("will decode _id in oneOf query", async () => {
await config.doInContext(config.appId, async () => {
const tableId = config.table!._id!
const searchParams: SearchParams = {
tableId,
query: {},
fields: ["name", "age"],
}
const result = await search(searchParams)
const searchParams: SearchParams = {
tableId,
query: {
oneOf: {
_id: ["%5B1%5D", "%5B4%5D", "%5B8%5D"],
},
},
}
const result = await search(searchParams)
expect(result.rows).toHaveLength(10)
expect(result.rows).toEqual(
expect.arrayContaining(
rows.map(r => ({
...expectAnyExternalColsAttributes,
name: r.name,
age: r.age,
}))
)
)
})
expect(result.rows).toHaveLength(3)
expect(result.rows.map(row => row.id)).toEqual([1, 4, 8])
})
})
})

1
packages/server/src/sdk/app/rows/search/utils.ts
View File

@ -1,6 +1,5 @@
import {
FieldType,
FieldTypeSubtypes,
SearchParams,
Table,
DocumentType,

14
packages/server/src/sdk/app/tables/external/index.ts vendored
View File

@ -3,6 +3,7 @@ import {
Operation,
RelationshipType,
RenameColumn,
AddColumn,
Table,
TableRequest,
ViewV2,
@ -32,7 +33,7 @@ import * as viewSdk from "../../views"
export async function save(
datasourceId: string,
update: Table,
opts?: { tableId?: string; renaming?: RenameColumn }
opts?: { tableId?: string; renaming?: RenameColumn; adding?: AddColumn }
) {
let tableToSave: TableRequest = {
...update,
@ -165,8 +166,17 @@ export async function save(
// remove the rename prop
delete tableToSave._rename
// if adding a new column, we need to rebuild the schema for that table to get the 'externalType' of the column
if (opts?.adding) {
datasource.entities[tableToSave.name] = (
await datasourceSdk.buildFilteredSchema(datasource, [tableToSave.name])
).tables[tableToSave.name]
} else {
datasource.entities[tableToSave.name] = tableToSave
}
// store it into couch now for budibase reference
datasource.entities[tableToSave.name] = tableToSave
await db.put(populateExternalTableSchemas(datasource))
// Since tables are stored inside datasources, we need to notify clients

10
packages/server/src/tests/utilities/TestConfiguration.ts
View File

@ -299,6 +299,16 @@ export default class TestConfiguration {
}
}
withUser(user: User, f: () => Promise<void>) {
const oldUser = this.user
this.user = user
try {
return f()
} finally {
this.user = oldUser
}
}
// UTILS
_req<Req extends Record<string, any> | void, Res>(

1
packages/types/src/api/web/index.ts
View File

@ -14,3 +14,4 @@ export * from "./cookies"
export * from "./automation"
export * from "./layout"
export * from "./query"
export * from "./role"

22
packages/types/src/api/web/role.ts Normal file
View File

@ -0,0 +1,22 @@
import { Role } from "../../documents"
export interface SaveRoleRequest {
_id?: string
_rev?: string
name: string
inherits: string
permissionId: string
version: string
}
export interface SaveRoleResponse extends Role {}
export interface FindRoleResponse extends Role {}
export type FetchRolesResponse = Role[]
export interface DestroyRoleResponse {
message: string
}
export type AccessibleRolesResponse = string[]

3
packages/types/src/documents/app/table/table.ts
View File

@ -1,6 +1,6 @@
import { Document } from "../../document"
import { View, ViewV2 } from "../view"
import { RenameColumn } from "../../../sdk"
import { AddColumn, RenameColumn } from "../../../sdk"
import { TableSchema } from "./schema"
export const INTERNAL_TABLE_SOURCE_ID = "bb_internal"
@ -29,5 +29,6 @@ export interface Table extends Document {
export interface TableRequest extends Table {
_rename?: RenameColumn
_add?: AddColumn
created?: boolean
}

4
packages/types/src/sdk/search.ts
View File

@ -60,6 +60,10 @@ export interface RenameColumn {
updated: string
}
export interface AddColumn {
name: string
}
export interface RelationshipsJson {
through?: string
from?: string