Merge branch 'cheeks-lab-day-binding-eval' of github.com:Budibase/budibase into cheeks-lab-day-binding-eval

This commit is contained in:
Andrew Kingston 2024-03-11 09:33:35 +00:00
commit 12b5989e57
14 changed files with 326 additions and 103 deletions

View File

@ -1,5 +1,5 @@
{
"version": "2.21.2",
"version": "2.21.3",
"npmClient": "yarn",
"packages": [
"packages/*",

View File

@ -84,16 +84,18 @@ export function getBuiltinRoles(): { [key: string]: RoleDoc } {
return cloneDeep(BUILTIN_ROLES)
}
export const BUILTIN_ROLE_ID_ARRAY = Object.values(BUILTIN_ROLES).map(
role => role._id
)
export function isBuiltin(role: string) {
return getBuiltinRole(role) !== undefined
}
export const BUILTIN_ROLE_NAME_ARRAY = Object.values(BUILTIN_ROLES).map(
role => role.name
)
export function isBuiltin(role?: string) {
return BUILTIN_ROLE_ID_ARRAY.some(builtin => role?.includes(builtin))
export function getBuiltinRole(roleId: string): Role | undefined {
const role = Object.values(BUILTIN_ROLES).find(role =>
roleId.includes(role._id)
)
if (!role) {
return undefined
}
return cloneDeep(role)
}
/**
@ -123,7 +125,7 @@ export function builtinRoleToNumber(id?: string) {
/**
* Converts any role to a number, but has to be async to get the roles from db.
*/
export async function roleToNumber(id?: string) {
export async function roleToNumber(id: string) {
if (isBuiltin(id)) {
return builtinRoleToNumber(id)
}
@ -131,7 +133,7 @@ export async function roleToNumber(id?: string) {
defaultPublic: true,
})) as RoleDoc[]
for (let role of hierarchy) {
if (isBuiltin(role?.inherits)) {
if (role?.inherits && isBuiltin(role.inherits)) {
return builtinRoleToNumber(role.inherits) + 1
}
}
@ -161,35 +163,28 @@ export function lowerBuiltinRoleID(roleId1?: string, roleId2?: string): string {
* @returns The role object, which may contain an "inherits" property.
*/
export async function getRole(
roleId?: string,
roleId: string,
opts?: { defaultPublic?: boolean }
): Promise<RoleDoc | undefined> {
if (!roleId) {
return undefined
}
let role: any = {}
): Promise<RoleDoc> {
// built in roles mostly come from the in-code implementation,
// but can be extended by a doc stored about them (e.g. permissions)
if (isBuiltin(roleId)) {
role = cloneDeep(
Object.values(BUILTIN_ROLES).find(role => role._id === roleId)
)
} else {
let role: RoleDoc | undefined = getBuiltinRole(roleId)
if (!role) {
// make sure has the prefix (if it has it then it won't be added)
roleId = prefixRoleID(roleId)
}
try {
const db = getAppDB()
const dbRole = await db.get(getDBRoleID(roleId))
role = Object.assign(role, dbRole)
const dbRole = await db.get<RoleDoc>(getDBRoleID(roleId))
role = Object.assign(role || {}, dbRole)
// finalise the ID
role._id = getExternalRoleID(role._id, role.version)
role._id = getExternalRoleID(role._id!, role.version)
} catch (err) {
if (!isBuiltin(roleId) && opts?.defaultPublic) {
return cloneDeep(BUILTIN_ROLES.PUBLIC)
}
// only throw an error if there is no role at all
if (Object.keys(role).length === 0) {
if (!role || Object.keys(role).length === 0) {
throw err
}
}
@ -200,7 +195,7 @@ export async function getRole(
* Simple function to get all the roles based on the top level user role ID.
*/
async function getAllUserRoles(
userRoleId?: string,
userRoleId: string,
opts?: { defaultPublic?: boolean }
): Promise<RoleDoc[]> {
// admins have access to all roles
@ -226,7 +221,7 @@ async function getAllUserRoles(
}
export async function getUserRoleIdHierarchy(
userRoleId?: string
userRoleId: string
): Promise<string[]> {
const roles = await getUserRoleHierarchy(userRoleId)
return roles.map(role => role._id!)
@ -241,7 +236,7 @@ export async function getUserRoleIdHierarchy(
* highest level of access and the last being the lowest level.
*/
export async function getUserRoleHierarchy(
userRoleId?: string,
userRoleId: string,
opts?: { defaultPublic?: boolean }
) {
// special case, if they don't have a role then they are a public user
@ -265,9 +260,9 @@ export function checkForRoleResourceArray(
return rolePerms
}
export async function getAllRoleIds(appId?: string) {
export async function getAllRoleIds(appId: string): Promise<string[]> {
const roles = await getAllRoles(appId)
return roles.map(role => role._id)
return roles.map(role => role._id!)
}
/**

View File

@ -7,8 +7,14 @@ import {
} from "@budibase/backend-core"
import { getUserMetadataParams, InternalTables } from "../../db/utils"
import {
AccessibleRolesResponse,
Database,
DestroyRoleResponse,
FetchRolesResponse,
FindRoleResponse,
Role,
SaveRoleRequest,
SaveRoleResponse,
UserCtx,
UserMetadata,
UserRoles,
@ -25,43 +31,36 @@ async function updateRolesOnUserTable(
db: Database,
roleId: string,
updateOption: string,
roleVersion: string | undefined
roleVersion?: string
) {
const table = await sdk.tables.getTable(InternalTables.USER_METADATA)
const schema = table.schema
const constraints = table.schema.roleId?.constraints
if (!constraints) {
return
}
const updatedRoleId =
roleVersion === roles.RoleIDVersion.NAME
? roles.getExternalRoleID(roleId, roleVersion)
: roleId
const indexOfRoleId = constraints.inclusion!.indexOf(updatedRoleId)
const remove = updateOption === UpdateRolesOptions.REMOVED
let updated = false
for (let prop of Object.keys(schema)) {
if (prop === "roleId") {
updated = true
const constraints = schema[prop].constraints!
const updatedRoleId =
roleVersion === roles.RoleIDVersion.NAME
? roles.getExternalRoleID(roleId, roleVersion)
: roleId
const indexOfRoleId = constraints.inclusion!.indexOf(updatedRoleId)
if (remove && indexOfRoleId !== -1) {
constraints.inclusion!.splice(indexOfRoleId, 1)
} else if (!remove && indexOfRoleId === -1) {
constraints.inclusion!.push(updatedRoleId)
}
break
}
}
if (updated) {
await db.put(table)
if (remove && indexOfRoleId !== -1) {
constraints.inclusion!.splice(indexOfRoleId, 1)
} else if (!remove && indexOfRoleId === -1) {
constraints.inclusion!.push(updatedRoleId)
}
await db.put(table)
}
export async function fetch(ctx: UserCtx) {
export async function fetch(ctx: UserCtx<void, FetchRolesResponse>) {
ctx.body = await roles.getAllRoles()
}
export async function find(ctx: UserCtx) {
export async function find(ctx: UserCtx<void, FindRoleResponse>) {
ctx.body = await roles.getRole(ctx.params.roleId)
}
export async function save(ctx: UserCtx) {
export async function save(ctx: UserCtx<SaveRoleRequest, SaveRoleResponse>) {
const db = context.getAppDB()
let { _id, name, inherits, permissionId, version } = ctx.request.body
let isCreate = false
@ -109,9 +108,9 @@ export async function save(ctx: UserCtx) {
ctx.body = role
}
export async function destroy(ctx: UserCtx) {
export async function destroy(ctx: UserCtx<void, DestroyRoleResponse>) {
const db = context.getAppDB()
let roleId = ctx.params.roleId
let roleId = ctx.params.roleId as string
if (roles.isBuiltin(roleId)) {
ctx.throw(400, "Cannot delete builtin role.")
} else {
@ -144,14 +143,18 @@ export async function destroy(ctx: UserCtx) {
ctx.status = 200
}
export async function accessible(ctx: UserCtx) {
export async function accessible(ctx: UserCtx<void, AccessibleRolesResponse>) {
let roleId = ctx.user?.roleId
if (!roleId) {
roleId = roles.BUILTIN_ROLE_IDS.PUBLIC
}
if (ctx.user && sharedSdk.users.isAdminOrBuilder(ctx.user)) {
const appId = context.getAppId()
ctx.body = await roles.getAllRoleIds(appId)
if (!appId) {
ctx.body = []
} else {
ctx.body = await roles.getAllRoleIds(appId)
}
} else {
ctx.body = await roles.getUserRoleIdHierarchy(roleId!)
}

View File

@ -63,7 +63,7 @@ export async function fetch(ctx: UserCtx) {
export async function clientFetch(ctx: UserCtx) {
const routing = await getRoutingStructure()
let roleId = ctx.user?.role?._id
const roleIds = await roles.getUserRoleIdHierarchy(roleId)
const roleIds = roleId ? await roles.getUserRoleIdHierarchy(roleId) : []
for (let topLevel of Object.values(routing.routes) as any) {
for (let subpathKey of Object.keys(topLevel.subpaths)) {
let found = false

View File

@ -1,12 +1,27 @@
import {
QueryJson,
SearchFilters,
Table,
Row,
Datasource,
DatasourcePlusQueryResponse,
Operation,
QueryJson,
Row,
SearchFilters,
} from "@budibase/types"
import { getDatasourceAndQuery } from "../../../sdk/app/rows/utils"
import { getSQLClient } from "../../../sdk/app/rows/utils"
import { cloneDeep } from "lodash"
import sdk from "../../../sdk"
import { makeExternalQuery } from "../../../integrations/base/query"
import { SqlClient } from "../../../integrations/utils"
const WRITE_OPERATIONS: Operation[] = [
Operation.CREATE,
Operation.UPDATE,
Operation.DELETE,
]
const DISABLED_WRITE_CLIENTS: SqlClient[] = [
SqlClient.MY_SQL,
SqlClient.MS_SQL,
SqlClient.ORACLE,
]
class CharSequence {
static alphabet = "abcdefghijklmnopqrstuvwxyz"
@ -43,6 +58,25 @@ export default class AliasTables {
this.charSeq = new CharSequence()
}
isAliasingEnabled(json: QueryJson, datasource: Datasource) {
const fieldLength = json.resource?.fields?.length
if (!fieldLength || fieldLength <= 0) {
return false
}
try {
const sqlClient = getSQLClient(datasource)
const isWrite = WRITE_OPERATIONS.includes(json.endpoint.operation)
const isDisabledClient = DISABLED_WRITE_CLIENTS.includes(sqlClient)
if (isWrite && isDisabledClient) {
return false
}
} catch (err) {
// if we can't get an SQL client, we can't alias
return false
}
return true
}
getAlias(tableName: string) {
if (this.aliases[tableName]) {
return this.aliases[tableName]
@ -111,8 +145,10 @@ export default class AliasTables {
}
async queryWithAliasing(json: QueryJson): DatasourcePlusQueryResponse {
const fieldLength = json.resource?.fields?.length
const aliasingEnabled = fieldLength && fieldLength > 0
const datasourceId = json.endpoint.datasourceId
const datasource = await sdk.datasources.get(datasourceId)
const aliasingEnabled = this.isAliasingEnabled(json, datasource)
if (aliasingEnabled) {
json = cloneDeep(json)
// run through the query json to update anywhere a table may be used
@ -158,7 +194,7 @@ export default class AliasTables {
}
json.tableAliases = invertedTableAliases
}
const response = await getDatasourceAndQuery(json)
const response = await makeExternalQuery(datasource, json)
if (Array.isArray(response) && aliasingEnabled) {
return this.reverse(response)
} else {

View File

@ -251,10 +251,15 @@ describe("/applications", () => {
describe("permissions", () => {
it("should only return apps a user has access to", async () => {
const user = await config.createUser()
const user = await config.createUser({
builder: { global: false },
admin: { global: false },
})
const apps = await config.api.application.fetch()
expect(apps.length).toBeGreaterThan(0)
await config.withUser(user, async () => {
const apps = await config.api.application.fetch()
expect(apps).toHaveLength(0)
})
})
})
})

View File

@ -1,5 +1,4 @@
import TestConfig from "../../../../tests/utilities/TestConfiguration"
import env from "../../../../environment"
import TestConfiguration from "../../../../tests/utilities/TestConfiguration"
import supertest from "supertest"
export * as structures from "../../../../tests/utilities/structures"
@ -47,10 +46,10 @@ export function delay(ms: number) {
}
let request: supertest.SuperTest<supertest.Test> | undefined | null,
config: TestConfig | null
config: TestConfiguration | null
export function beforeAll() {
config = new TestConfig()
config = new TestConfiguration()
request = config.getRequest()
}

View File

@ -435,10 +435,12 @@ class InternalBuilder {
aliases?: QueryJson["tableAliases"]
): Knex.QueryBuilder {
const tableName = endpoint.entityId
const tableAliased = aliases?.[tableName]
? `${tableName} as ${aliases?.[tableName]}`
: tableName
let query = knex(tableAliased)
const tableAlias = aliases?.[tableName]
let table: string | Record<string, string> = tableName
if (tableAlias) {
table = { [tableAlias]: tableName }
}
let query = knex(table)
if (endpoint.schema) {
query = query.withSchema(endpoint.schema)
}

View File

@ -14,7 +14,12 @@ import firebase from "./firebase"
import redis from "./redis"
import snowflake from "./snowflake"
import oracle from "./oracle"
import { SourceName, Integration, PluginType } from "@budibase/types"
import {
SourceName,
Integration,
PluginType,
IntegrationBase,
} from "@budibase/types"
import { getDatasourcePlugin } from "../utilities/fileSystem"
import env from "../environment"
import cloneDeep from "lodash/cloneDeep"
@ -40,25 +45,28 @@ const DEFINITIONS: Record<SourceName, Integration | undefined> = {
[SourceName.BUDIBASE]: undefined,
}
const INTEGRATIONS: Record<SourceName, any> = {
[SourceName.POSTGRES]: postgres.integration,
[SourceName.DYNAMODB]: dynamodb.integration,
[SourceName.MONGODB]: mongodb.integration,
[SourceName.ELASTICSEARCH]: elasticsearch.integration,
[SourceName.COUCHDB]: couchdb.integration,
[SourceName.SQL_SERVER]: sqlServer.integration,
[SourceName.S3]: s3.integration,
[SourceName.AIRTABLE]: airtable.integration,
[SourceName.MYSQL]: mysql.integration,
[SourceName.ARANGODB]: arangodb.integration,
[SourceName.REST]: rest.integration,
[SourceName.FIRESTORE]: firebase.integration,
[SourceName.GOOGLE_SHEETS]: googlesheets.integration,
[SourceName.REDIS]: redis.integration,
[SourceName.SNOWFLAKE]: snowflake.integration,
[SourceName.ORACLE]: undefined,
[SourceName.BUDIBASE]: undefined,
}
type IntegrationBaseConstructor = new (...args: any[]) => IntegrationBase
const INTEGRATIONS: Record<SourceName, IntegrationBaseConstructor | undefined> =
{
[SourceName.POSTGRES]: postgres.integration,
[SourceName.DYNAMODB]: dynamodb.integration,
[SourceName.MONGODB]: mongodb.integration,
[SourceName.ELASTICSEARCH]: elasticsearch.integration,
[SourceName.COUCHDB]: couchdb.integration,
[SourceName.SQL_SERVER]: sqlServer.integration,
[SourceName.S3]: s3.integration,
[SourceName.AIRTABLE]: airtable.integration,
[SourceName.MYSQL]: mysql.integration,
[SourceName.ARANGODB]: arangodb.integration,
[SourceName.REST]: rest.integration,
[SourceName.FIRESTORE]: firebase.integration,
[SourceName.GOOGLE_SHEETS]: googlesheets.integration,
[SourceName.REDIS]: redis.integration,
[SourceName.SNOWFLAKE]: snowflake.integration,
[SourceName.ORACLE]: undefined,
[SourceName.BUDIBASE]: undefined,
}
// optionally add oracle integration if the oracle binary can be installed
if (

View File

@ -1,4 +1,4 @@
import { QueryJson } from "@budibase/types"
import { Datasource, Operation, QueryJson, SourceName } from "@budibase/types"
import { join } from "path"
import Sql from "../base/sql"
import { SqlClient } from "../utils"
@ -198,6 +198,114 @@ describe("Captures of real examples", () => {
})
})
describe("check aliasing is disabled/enabled", () => {
const tables = ["tableA", "tableB"]
function getDatasource(source: SourceName): Datasource {
return {
source,
type: "datasource",
isSQL: true,
}
}
function getQuery(op: Operation, fields: string[] = ["a"]): QueryJson {
return {
endpoint: { datasourceId: "", entityId: "", operation: op },
resource: {
fields,
},
}
}
it("should check for Postgres aliased status", () => {
const aliasing = new AliasTables(tables)
const datasource = getDatasource(SourceName.POSTGRES)
expect(
aliasing.isAliasingEnabled(getQuery(Operation.CREATE), datasource)
).toEqual(true)
expect(
aliasing.isAliasingEnabled(getQuery(Operation.READ), datasource)
).toEqual(true)
expect(
aliasing.isAliasingEnabled(getQuery(Operation.UPDATE), datasource)
).toEqual(true)
expect(
aliasing.isAliasingEnabled(getQuery(Operation.DELETE), datasource)
).toEqual(true)
})
it("should check for MS-SQL aliased status", () => {
const aliasing = new AliasTables(tables)
const datasource = getDatasource(SourceName.SQL_SERVER)
expect(
aliasing.isAliasingEnabled(getQuery(Operation.CREATE), datasource)
).toEqual(false)
expect(
aliasing.isAliasingEnabled(getQuery(Operation.READ), datasource)
).toEqual(true)
expect(
aliasing.isAliasingEnabled(getQuery(Operation.UPDATE), datasource)
).toEqual(false)
expect(
aliasing.isAliasingEnabled(getQuery(Operation.DELETE), datasource)
).toEqual(false)
})
it("should check for MySQL aliased status", () => {
const aliasing = new AliasTables(tables)
const datasource = getDatasource(SourceName.MYSQL)
expect(
aliasing.isAliasingEnabled(getQuery(Operation.CREATE), datasource)
).toEqual(false)
expect(
aliasing.isAliasingEnabled(getQuery(Operation.READ), datasource)
).toEqual(true)
expect(
aliasing.isAliasingEnabled(getQuery(Operation.UPDATE), datasource)
).toEqual(false)
expect(
aliasing.isAliasingEnabled(getQuery(Operation.DELETE), datasource)
).toEqual(false)
})
it("should check for Oracle aliased status", () => {
const aliasing = new AliasTables(tables)
const datasource = getDatasource(SourceName.ORACLE)
expect(
aliasing.isAliasingEnabled(getQuery(Operation.CREATE), datasource)
).toEqual(false)
expect(
aliasing.isAliasingEnabled(getQuery(Operation.READ), datasource)
).toEqual(true)
expect(
aliasing.isAliasingEnabled(getQuery(Operation.UPDATE), datasource)
).toEqual(false)
expect(
aliasing.isAliasingEnabled(getQuery(Operation.DELETE), datasource)
).toEqual(false)
})
it("should disable aliasing for non-SQL datasources", () => {
const aliasing = new AliasTables(tables)
expect(
aliasing.isAliasingEnabled(getQuery(Operation.READ), {
source: SourceName.GOOGLE_SHEETS,
type: "datasource",
isSQL: false,
})
)
})
it("should disable when no fields", () => {
const aliasing = new AliasTables(tables)
const datasource = getDatasource(SourceName.POSTGRES)
expect(
aliasing.isAliasingEnabled(getQuery(Operation.READ, []), datasource)
).toEqual(false)
})
})
describe("check some edge cases", () => {
const tableNames = ["hello", "world"]

View File

@ -1,17 +1,51 @@
import cloneDeep from "lodash/cloneDeep"
import validateJs from "validate.js"
import {
Datasource,
DatasourcePlusQueryResponse,
FieldType,
QueryJson,
Row,
SourceName,
Table,
TableSchema,
DatasourcePlusQueryResponse,
} from "@budibase/types"
import { makeExternalQuery } from "../../../integrations/base/query"
import { Format } from "../../../api/controllers/view/exporters"
import sdk from "../.."
import { isRelationshipColumn } from "../../../db/utils"
import { SqlClient } from "../../../integrations/utils"
const SQL_CLIENT_SOURCE_MAP: Record<SourceName, SqlClient | undefined> = {
[SourceName.POSTGRES]: SqlClient.POSTGRES,
[SourceName.MYSQL]: SqlClient.MY_SQL,
[SourceName.SQL_SERVER]: SqlClient.MS_SQL,
[SourceName.ORACLE]: SqlClient.ORACLE,
[SourceName.DYNAMODB]: undefined,
[SourceName.MONGODB]: undefined,
[SourceName.ELASTICSEARCH]: undefined,
[SourceName.COUCHDB]: undefined,
[SourceName.S3]: undefined,
[SourceName.AIRTABLE]: undefined,
[SourceName.ARANGODB]: undefined,
[SourceName.REST]: undefined,
[SourceName.FIRESTORE]: undefined,
[SourceName.GOOGLE_SHEETS]: undefined,
[SourceName.REDIS]: undefined,
[SourceName.SNOWFLAKE]: undefined,
[SourceName.BUDIBASE]: undefined,
}
export function getSQLClient(datasource: Datasource): SqlClient {
if (!datasource.isSQL) {
throw new Error("Cannot get SQL Client for non-SQL datasource")
}
const lookup = SQL_CLIENT_SOURCE_MAP[datasource.source]
if (lookup) {
return lookup
}
throw new Error("Unable to determine client for SQL datasource")
}
export async function getDatasourceAndQuery(
json: QueryJson

View File

@ -299,6 +299,16 @@ export default class TestConfiguration {
}
}
withUser(user: User, f: () => Promise<void>) {
const oldUser = this.user
this.user = user
try {
return f()
} finally {
this.user = oldUser
}
}
// UTILS
_req<Req extends Record<string, any> | void, Res>(

View File

@ -14,3 +14,4 @@ export * from "./cookies"
export * from "./automation"
export * from "./layout"
export * from "./query"
export * from "./role"

View File

@ -0,0 +1,22 @@
import { Role } from "../../documents"
export interface SaveRoleRequest {
_id?: string
_rev?: string
name: string
inherits: string
permissionId: string
version: string
}
export interface SaveRoleResponse extends Role {}
export interface FindRoleResponse extends Role {}
export type FetchRolesResponse = Role[]
export interface DestroyRoleResponse {
message: string
}
export type AccessibleRolesResponse = string[]