Merge branch 'v3-ui' of github.com:Budibase/budibase into view-calculation-ui

This commit is contained in:
Andrew Kingston 2024-10-08 14:06:52 +01:00
commit 491f74e6bc
No known key found for this signature in database
26 changed files with 794 additions and 175 deletions

View File

@ -184,6 +184,10 @@ spec:
- name: NODE_DEBUG
value: {{ .Values.services.apps.nodeDebug | quote }}
{{ end }}
{{ if .Values.services.apps.xssSafeMode }}
- name: XSS_SAFE_MODE
value: {{ .Values.services.apps.xssSafeMode | quote }}
{{ end }}
{{ if .Values.globals.datadogApmEnabled }}
- name: DD_LOGS_INJECTION
value: {{ .Values.globals.datadogApmEnabled | quote }}

View File

@ -1,6 +1,6 @@
{
"$schema": "node_modules/lerna/schemas/lerna-schema.json",
"version": "2.32.11",
"version": "2.32.12",
"npmClient": "yarn",
"packages": [
"packages/*",

@ -1 +1 @@
Subproject commit 3e24f6293ff5ee5f9b42822e001504e3bbf19cc0
Subproject commit 8cd052ce8288f343812a514d06c5a9459b3ba1a8

View File

@ -371,11 +371,21 @@ export class DatabaseImpl implements Database {
return this.performCall(() => {
return async () => {
const response = await directCouchUrlCall(args)
const json = await response.json()
const text = await response.text()
if (response.status > 300) {
let json
try {
json = JSON.parse(text)
} catch (err) {
console.error(`SQS error: ${text}`)
throw new CouchDBError(
"error while running SQS query, please try again later",
{ name: "sqs_error", status: response.status }
)
}
throw json
}
return json as T
return JSON.parse(text) as T
}
})
}

View File

@ -2,7 +2,6 @@ import { generateGlobalUserID } from "../../../db"
import { authError } from "../utils"
import * as users from "../../../users"
import * as context from "../../../context"
import fetch from "node-fetch"
import {
SaveSSOUserFunction,
SSOAuthDetails,
@ -97,28 +96,13 @@ export async function authenticate(
return done(null, ssoUser)
}
async function getProfilePictureUrl(user: User, details: SSOAuthDetails) {
const pictureUrl = details.profile?._json.picture
if (pictureUrl) {
const response = await fetch(pictureUrl)
if (response.status === 200) {
const type = response.headers.get("content-type") as string
if (type.startsWith("image/")) {
return pictureUrl
}
}
}
}
/**
* @returns a user that has been sync'd with third party information
*/
async function syncUser(user: User, details: SSOAuthDetails): Promise<SSOUser> {
let firstName
let lastName
let pictureUrl
let oauth2
let thirdPartyProfile
if (details.profile) {
const profile = details.profile
@ -134,12 +118,6 @@ async function syncUser(user: User, details: SSOAuthDetails): Promise<SSOUser> {
lastName = name.familyName
}
}
pictureUrl = await getProfilePictureUrl(user, details)
thirdPartyProfile = {
...profile._json,
}
}
// oauth tokens for future use
@ -155,8 +133,6 @@ async function syncUser(user: User, details: SSOAuthDetails): Promise<SSOUser> {
providerType: details.providerType,
firstName,
lastName,
thirdPartyProfile,
pictureUrl,
oauth2,
}
}

View File

@ -6,9 +6,6 @@ import {
AccountSSOProviderType,
AuthType,
CloudAccount,
CreateAccount,
CreatePassswordAccount,
CreateVerifiableSSOAccount,
Hosting,
SSOAccount,
} from "@budibase/types"
@ -19,6 +16,7 @@ export const account = (partial: Partial<Account> = {}): Account => {
accountId: uuid(),
tenantId: generator.word(),
email: generator.email({ domain: "example.com" }),
accountName: generator.word(),
tenantName: generator.word(),
hosting: Hosting.SELF,
createdAt: Date.now(),
@ -61,10 +59,8 @@ export function ssoAccount(account: Account = cloudAccount()): SSOAccount {
accessToken: generator.string(),
refreshToken: generator.string(),
},
pictureUrl: generator.url(),
provider: provider(),
providerType: providerType(),
thirdPartyProfile: {},
}
}
@ -78,68 +74,7 @@ export function verifiableSsoAccount(
accessToken: generator.string(),
refreshToken: generator.string(),
},
pictureUrl: generator.url(),
provider: AccountSSOProvider.MICROSOFT,
providerType: AccountSSOProviderType.MICROSOFT,
thirdPartyProfile: { id: "abc123" },
}
}
export const cloudCreateAccount: CreatePassswordAccount = {
email: "cloud@budibase.com",
tenantId: "cloud",
hosting: Hosting.CLOUD,
authType: AuthType.PASSWORD,
password: "Password123!",
tenantName: "cloud",
name: "Budi Armstrong",
size: "10+",
profession: "Software Engineer",
}
export const cloudSSOCreateAccount: CreateAccount = {
email: "cloud-sso@budibase.com",
tenantId: "cloud-sso",
hosting: Hosting.CLOUD,
authType: AuthType.SSO,
tenantName: "cloudsso",
name: "Budi Armstrong",
size: "10+",
profession: "Software Engineer",
}
export const cloudVerifiableSSOCreateAccount: CreateVerifiableSSOAccount = {
email: "cloud-sso@budibase.com",
tenantId: "cloud-sso",
hosting: Hosting.CLOUD,
authType: AuthType.SSO,
tenantName: "cloudsso",
name: "Budi Armstrong",
size: "10+",
profession: "Software Engineer",
provider: AccountSSOProvider.MICROSOFT,
thirdPartyProfile: { id: "abc123" },
}
export const selfCreateAccount: CreatePassswordAccount = {
email: "self@budibase.com",
tenantId: "self",
hosting: Hosting.SELF,
authType: AuthType.PASSWORD,
password: "Password123!",
tenantName: "self",
name: "Budi Armstrong",
size: "10+",
profession: "Software Engineer",
}
export const selfSSOCreateAccount: CreateAccount = {
email: "self-sso@budibase.com",
tenantId: "self-sso",
hosting: Hosting.SELF,
authType: AuthType.SSO,
tenantName: "selfsso",
name: "Budi Armstrong",
size: "10+",
profession: "Software Engineer",
}

View File

@ -25,7 +25,6 @@ export const user = (userProps?: Partial<Omit<User, "userId">>): User => {
roles: { app_test: "admin" },
firstName: generator.first(),
lastName: generator.last(),
pictureUrl: "http://example.com",
tenantId: tenant.id(),
...userProps,
}
@ -86,9 +85,5 @@ export function ssoUser(
oauth2: opts.details?.oauth2,
provider: opts.details?.provider!,
providerType: opts.details?.providerType!,
thirdPartyProfile: {
email: base.email,
picture: base.pictureUrl,
},
}
}

View File

@ -127,13 +127,13 @@ export async function create(ctx: Ctx<CreateViewRequest, ViewResponse>) {
const parsedView: Omit<RequiredKeys<ViewV2>, "id" | "version"> = {
name: view.name,
type: view.type,
tableId: view.tableId,
query: view.query,
queryUI: view.queryUI,
sort: view.sort,
schema,
primaryDisplay: view.primaryDisplay,
uiMetadata: view.uiMetadata,
}
const result = await sdk.views.create(tableId, parsedView)
ctx.status = 201
@ -163,6 +163,7 @@ export async function update(ctx: Ctx<UpdateViewRequest, ViewResponse>) {
const parsedView: RequiredKeys<ViewV2> = {
id: view.id,
name: view.name,
type: view.type,
version: view.version,
tableId: view.tableId,
query: view.query,
@ -170,7 +171,6 @@ export async function update(ctx: Ctx<UpdateViewRequest, ViewResponse>) {
sort: view.sort,
schema,
primaryDisplay: view.primaryDisplay,
uiMetadata: view.uiMetadata,
}
const result = await sdk.views.update(tableId, parsedView)

View File

@ -26,6 +26,7 @@ import {
BBReferenceFieldSubType,
NumericCalculationFieldMetadata,
ViewV2Schema,
ViewV2Type,
} from "@budibase/types"
import { generator, mocks } from "@budibase/backend-core/tests"
import { DatabaseName, getDatasource } from "../../../integrations/tests/utils"
@ -155,7 +156,7 @@ describe.each([
})
it("can persist views with all fields", async () => {
const newView: Required<Omit<CreateViewRequest, "queryUI">> = {
const newView: Required<Omit<CreateViewRequest, "queryUI" | "type">> = {
name: generator.name(),
tableId: table._id!,
primaryDisplay: "id",
@ -177,9 +178,6 @@ describe.each([
visible: true,
},
},
uiMetadata: {
foo: "bar",
},
}
const res = await config.api.viewV2.create(newView)
@ -549,6 +547,7 @@ describe.each([
let view = await config.api.viewV2.create({
tableId: table._id!,
name: generator.guid(),
type: ViewV2Type.CALCULATION,
schema: {
sum: {
visible: true,
@ -571,6 +570,184 @@ describe.each([
expect(sum.calculationType).toEqual(CalculationType.SUM)
expect(sum.field).toEqual("Price")
})
it("cannot create a view with calculation fields unless it has the right type", async () => {
await config.api.viewV2.create(
{
tableId: table._id!,
name: generator.guid(),
schema: {
sum: {
visible: true,
calculationType: CalculationType.SUM,
field: "Price",
},
},
},
{
status: 400,
body: {
message:
"Calculation fields are not allowed in non-calculation views",
},
}
)
})
it("cannot create a calculation view with more than 5 aggregations", async () => {
await config.api.viewV2.create(
{
tableId: table._id!,
name: generator.guid(),
type: ViewV2Type.CALCULATION,
schema: {
sum: {
visible: true,
calculationType: CalculationType.SUM,
field: "Price",
},
count: {
visible: true,
calculationType: CalculationType.COUNT,
field: "Price",
},
min: {
visible: true,
calculationType: CalculationType.MIN,
field: "Price",
},
max: {
visible: true,
calculationType: CalculationType.MAX,
field: "Price",
},
avg: {
visible: true,
calculationType: CalculationType.AVG,
field: "Price",
},
sum2: {
visible: true,
calculationType: CalculationType.SUM,
field: "Price",
},
},
},
{
status: 400,
body: {
message: "Calculation views can only have a maximum of 5 fields",
},
}
)
})
it("cannot create a calculation view with duplicate calculations", async () => {
await config.api.viewV2.create(
{
tableId: table._id!,
name: generator.guid(),
type: ViewV2Type.CALCULATION,
schema: {
sum: {
visible: true,
calculationType: CalculationType.SUM,
field: "Price",
},
sum2: {
visible: true,
calculationType: CalculationType.SUM,
field: "Price",
},
},
},
{
status: 400,
body: {
message:
'Duplicate calculation on field "Price", calculation type "sum"',
},
}
)
})
it("finds duplicate counts", async () => {
await config.api.viewV2.create(
{
tableId: table._id!,
name: generator.guid(),
type: ViewV2Type.CALCULATION,
schema: {
count: {
visible: true,
calculationType: CalculationType.COUNT,
},
count2: {
visible: true,
calculationType: CalculationType.COUNT,
},
},
},
{
status: 400,
body: {
message:
'Duplicate calculation on field "*", calculation type "count"',
},
}
)
})
it("finds duplicate count distincts", async () => {
await config.api.viewV2.create(
{
tableId: table._id!,
name: generator.guid(),
type: ViewV2Type.CALCULATION,
schema: {
count: {
visible: true,
calculationType: CalculationType.COUNT,
distinct: true,
field: "Price",
},
count2: {
visible: true,
calculationType: CalculationType.COUNT,
distinct: true,
field: "Price",
},
},
},
{
status: 400,
body: {
message:
'Duplicate calculation on field "Price", calculation type "count"',
},
}
)
})
it("does not confuse counts and count distincts in the duplicate check", async () => {
await config.api.viewV2.create({
tableId: table._id!,
name: generator.guid(),
type: ViewV2Type.CALCULATION,
schema: {
count: {
visible: true,
calculationType: CalculationType.COUNT,
},
count2: {
visible: true,
calculationType: CalculationType.COUNT,
distinct: true,
field: "Price",
},
},
})
})
})
describe("update", () => {
@ -615,7 +792,9 @@ describe.each([
it("can update all fields", async () => {
const tableId = table._id!
const updatedData: Required<Omit<UpdateViewRequest, "queryUI">> = {
const updatedData: Required<
Omit<UpdateViewRequest, "queryUI" | "type">
> = {
version: view.version,
id: view.id,
tableId,
@ -643,9 +822,6 @@ describe.each([
readonly: true,
},
},
uiMetadata: {
foo: "bar",
},
}
await config.api.viewV2.update(updatedData)
@ -830,6 +1006,32 @@ describe.each([
)
})
it("cannot update view type after creation", async () => {
const view = await config.api.viewV2.create({
tableId: table._id!,
name: generator.guid(),
schema: {
id: { visible: true },
Price: {
visible: true,
},
},
})
await config.api.viewV2.update(
{
...view,
type: ViewV2Type.CALCULATION,
},
{
status: 400,
body: {
message: "Cannot update view type after creation",
},
}
)
})
isInternal &&
it("updating schema will only validate modified field", async () => {
let view = await config.api.viewV2.create({
@ -902,6 +1104,7 @@ describe.each([
view = await config.api.viewV2.create({
tableId: table._id!,
name: generator.guid(),
type: ViewV2Type.CALCULATION,
schema: {
country: {
visible: true,
@ -1024,6 +1227,26 @@ describe.each([
)
})
it("can add a new group by field that is invisible, even if required on the table", async () => {
view.schema!.name = { visible: false }
await config.api.viewV2.update(view)
const { rows } = await config.api.row.search(view.id)
expect(rows).toHaveLength(2)
expect(rows).toEqual(
expect.arrayContaining([
{
country: "USA",
age: 65,
},
{
country: "UK",
age: 61,
},
])
)
})
it("can add a new calculation field", async () => {
view.schema!.count = {
visible: true,
@ -2639,6 +2862,7 @@ describe.each([
it("should be able to search by calculations", async () => {
const view = await config.api.viewV2.create({
tableId: table._id!,
type: ViewV2Type.CALCULATION,
name: generator.guid(),
schema: {
"Quantity Sum": {
@ -2673,6 +2897,7 @@ describe.each([
const view = await config.api.viewV2.create({
tableId: table._id!,
name: generator.guid(),
type: ViewV2Type.CALCULATION,
schema: {
quantity: {
visible: true,
@ -2711,6 +2936,7 @@ describe.each([
const view = await config.api.viewV2.create({
tableId: table._id!,
name: generator.guid(),
type: ViewV2Type.CALCULATION,
schema: {
aggregate: {
visible: true,
@ -2771,6 +2997,7 @@ describe.each([
const view = await config.api.viewV2.create({
tableId: table._id!,
name: generator.guid(),
type: ViewV2Type.CALCULATION,
schema: {
count: {
visible: true,
@ -2805,6 +3032,7 @@ describe.each([
{
tableId: table._id!,
name: generator.guid(),
type: ViewV2Type.CALCULATION,
schema: {
count: {
visible: true,
@ -2853,6 +3081,7 @@ describe.each([
const view = await config.api.viewV2.create({
tableId: table._id!,
name: generator.guid(),
type: ViewV2Type.CALCULATION,
schema: {
sum: {
visible: true,

View File

@ -83,6 +83,7 @@ const environment = {
PLUGINS_DIR: process.env.PLUGINS_DIR || DEFAULTS.PLUGINS_DIR,
MAX_IMPORT_SIZE_MB: process.env.MAX_IMPORT_SIZE_MB,
SESSION_EXPIRY_SECONDS: process.env.SESSION_EXPIRY_SECONDS,
XSS_SAFE_MODE: process.env.XSS_SAFE_MODE,
// SQL
SQL_MAX_ROWS: process.env.SQL_MAX_ROWS,
SQL_LOGGING_ENABLE: process.env.SQL_LOGGING_ENABLE,

View File

@ -8,7 +8,6 @@ import { init } from ".."
import TestConfiguration from "../../tests/utilities/TestConfiguration"
const DATE = "2021-01-21T12:00:00"
tk.freeze(DATE)
describe("jsRunner (using isolated-vm)", () => {
@ -48,6 +47,33 @@ describe("jsRunner (using isolated-vm)", () => {
).toEqual("ReferenceError: process is not defined")
})
it("should not allow the context to be mutated", async () => {
const context = { array: [1] }
const result = await processJS(
`
const array = $("array");
array.push(2);
return array[1]
`,
context
)
expect(result).toEqual(2)
expect(context.array).toEqual([1])
})
it("should copy values whenever returning them from $", async () => {
const context = { array: [1] }
const result = await processJS(
`
$("array").push(2);
return $("array")[1];
`,
context
)
expect(result).toEqual(undefined)
expect(context.array).toEqual([1])
})
describe("helpers", () => {
runJsHelpersTests({
funcWrap: (func: any) => config.doInContext(config.getAppId(), func),

View File

@ -8,6 +8,7 @@ import {
import { generateTableID } from "../../../../db/utils"
import { validate } from "../utils"
import { generator } from "@budibase/backend-core/tests"
import { withEnv } from "../../../../environment"
describe("validate", () => {
const hour = () => generator.hour().toString().padStart(2, "0")
@ -332,4 +333,46 @@ describe("validate", () => {
})
})
})
describe("XSS Safe mode", () => {
const getTable = (): Table => ({
type: "table",
_id: generateTableID(),
name: "table",
sourceId: INTERNAL_TABLE_SOURCE_ID,
sourceType: TableSourceType.INTERNAL,
schema: {
text: {
name: "sometext",
type: FieldType.STRING,
},
},
})
it.each([
"SELECT * FROM users WHERE username = 'admin' --",
"SELECT * FROM users WHERE id = 1; DROP TABLE users;",
"1' OR '1' = '1",
"' OR 'a' = 'a",
"<script>alert('XSS');</script>",
'"><img src=x onerror=alert(1)>',
"</script><script>alert('test')</script>",
"<div onmouseover=\"alert('XSS')\">Hover over me!</div>",
"'; EXEC sp_msforeachtable 'DROP TABLE ?'; --",
"{alert('Injected')}",
"UNION SELECT * FROM users",
"INSERT INTO users (username, password) VALUES ('admin', 'password')",
"/* This is a comment */ SELECT * FROM users",
'<iframe src="http://malicious-site.com"></iframe>',
])("test potentially unsafe input: %s", async input => {
await withEnv({ XSS_SAFE_MODE: "1" }, async () => {
const table = getTable()
const row = { text: input }
const output = await validate({ source: table, row })
expect(output.valid).toBe(false)
expect(output.errors).toStrictEqual({
text: ["Input not sanitised - potentially vulnerable to XSS"],
})
})
})
})
})

View File

@ -22,6 +22,7 @@ import { extractViewInfoFromID, isRelationshipColumn } from "../../../db/utils"
import { isSQL } from "../../../integrations/utils"
import { docIds, sql } from "@budibase/backend-core"
import { getTableFromSource } from "../../../api/controllers/row/utils"
import env from "../../../environment"
const SQL_CLIENT_SOURCE_MAP: Record<SourceName, SqlClient | undefined> = {
[SourceName.POSTGRES]: SqlClient.POSTGRES,
@ -43,6 +44,9 @@ const SQL_CLIENT_SOURCE_MAP: Record<SourceName, SqlClient | undefined> = {
[SourceName.BUDIBASE]: undefined,
}
const XSS_INPUT_REGEX =
/[<>;"'(){}]|--|\/\*|\*\/|union|select|insert|drop|delete|update|exec|script/i
export function getSQLClient(datasource: Datasource): SqlClient {
if (!isSQL(datasource)) {
throw new Error("Cannot get SQL Client for non-SQL datasource")
@ -222,6 +226,15 @@ export async function validate({
} else {
res = validateJs.single(row[fieldName], constraints)
}
if (env.XSS_SAFE_MODE && typeof row[fieldName] === "string") {
if (XSS_INPUT_REGEX.test(row[fieldName])) {
errors[fieldName] = [
"Input not sanitised - potentially vulnerable to XSS",
]
}
}
if (res) errors[fieldName] = res
}
return { valid: Object.keys(errors).length === 0, errors }

View File

@ -70,6 +70,9 @@ export async function update(tableId: string, view: ViewV2): Promise<ViewV2> {
if (!existingView || !existingView.name) {
throw new HTTPError(`View ${view.id} not found in table ${tableId}`, 404)
}
if (isV2(existingView) && existingView.type !== view.type) {
throw new HTTPError(`Cannot update view type after creation`, 400)
}
delete views[existingView.name]
views[view.name] = view

View File

@ -62,11 +62,31 @@ async function guardCalculationViewSchema(
view: Omit<ViewV2, "id" | "version">
) {
const calculationFields = helpers.views.calculationFields(view)
for (const calculationFieldName of Object.keys(calculationFields)) {
const schema = calculationFields[calculationFieldName]
if (Object.keys(calculationFields).length > 5) {
throw new HTTPError(
"Calculation views can only have a maximum of 5 fields",
400
)
}
const seen: Record<string, Record<CalculationType, boolean>> = {}
for (const name of Object.keys(calculationFields)) {
const schema = calculationFields[name]
const isCount = schema.calculationType === CalculationType.COUNT
const isDistinct = isCount && "distinct" in schema && schema.distinct
const field = isCount && !isDistinct ? "*" : schema.field
if (seen[field]?.[schema.calculationType]) {
throw new HTTPError(
`Duplicate calculation on field "${field}", calculation type "${schema.calculationType}"`,
400
)
}
seen[field] ??= {} as Record<CalculationType, boolean>
seen[field][schema.calculationType] = true
// Count fields that aren't distinct don't need to reference another field,
// so we don't validate it.
if (isCount && !isDistinct) {
@ -76,14 +96,14 @@ async function guardCalculationViewSchema(
const targetSchema = table.schema[schema.field]
if (!targetSchema) {
throw new HTTPError(
`Calculation field "${calculationFieldName}" references field "${schema.field}" which does not exist in the table schema`,
`Calculation field "${name}" references field "${schema.field}" which does not exist in the table schema`,
400
)
}
if (!isCount && !helpers.schema.isNumeric(targetSchema)) {
throw new HTTPError(
`Calculation field "${calculationFieldName}" references field "${schema.field}" which is not a numeric field`,
`Calculation field "${name}" references field "${schema.field}" which is not a numeric field`,
400
)
}
@ -109,10 +129,21 @@ async function guardViewSchema(
if (helpers.views.isCalculationView(view)) {
await guardCalculationViewSchema(table, view)
} else {
if (helpers.views.hasCalculationFields(view)) {
throw new HTTPError(
"Calculation fields are not allowed in non-calculation views",
400
)
}
}
await checkReadonlyFields(table, view)
checkRequiredFields(table, view)
if (!helpers.views.isCalculationView(view)) {
checkRequiredFields(table, view)
}
checkDisplayField(view)
}

View File

@ -59,6 +59,10 @@ export async function update(tableId: string, view: ViewV2): Promise<ViewV2> {
throw new HTTPError(`View ${view.id} not found in table ${tableId}`, 404)
}
if (isV2(existingView) && existingView.type !== view.type) {
throw new HTTPError(`Cannot update view type after creation`, 400)
}
delete table.views[existingView.name]
table.views[view.name] = view
await db.put(table)

View File

@ -3,6 +3,7 @@ import {
ViewCalculationFieldMetadata,
ViewFieldMetadata,
ViewV2,
ViewV2Type,
} from "@budibase/types"
import { pickBy } from "lodash"
@ -21,6 +22,10 @@ export function isBasicViewField(
type UnsavedViewV2 = Omit<ViewV2, "id" | "version">
export function isCalculationView(view: UnsavedViewV2) {
return view.type === ViewV2Type.CALCULATION
}
export function hasCalculationFields(view: UnsavedViewV2) {
return Object.values(view.schema || {}).some(isCalculationField)
}

View File

@ -1,13 +1,14 @@
import { atob, isJSAllowed } from "../utilities"
import cloneDeep from "lodash/fp/cloneDeep"
import { atob, isBackendService, isJSAllowed } from "../utilities"
import { LITERAL_MARKER } from "../helpers/constants"
import { getJsHelperList } from "./list"
import { iifeWrapper } from "../iife"
import { JsTimeoutError, UserScriptError } from "../errors"
import { cloneDeep } from "lodash/fp"
// The method of executing JS scripts depends on the bundle being built.
// This setter is used in the entrypoint (either index.js or index.mjs).
let runJS: ((js: string, context: any) => any) | undefined = undefined
let runJS: ((js: string, context: Record<string, any>) => any) | undefined =
undefined
export const setJSRunner = (runner: typeof runJS) => (runJS = runner)
export const removeJSRunner = () => {
@ -31,9 +32,19 @@ const removeSquareBrackets = (value: string) => {
return value
}
const isReservedKey = (key: string) =>
key === "snippets" ||
key === "helpers" ||
key.startsWith("snippets.") ||
key.startsWith("helpers.")
// Our context getter function provided to JS code as $.
// Extracts a value from context.
const getContextValue = (path: string, context: any) => {
// We populate `snippets` ourselves, don't allow access to it.
if (isReservedKey(path)) {
return undefined
}
const literalStringRegex = /^(["'`]).*\1$/
let data = context
// check if it's a literal string - just return path if its quoted
@ -46,6 +57,7 @@ const getContextValue = (path: string, context: any) => {
}
data = data[removeSquareBrackets(key)]
})
return data
}
@ -67,10 +79,23 @@ export function processJS(handlebars: string, context: any) {
snippetMap[snippet.name] = snippet.code
}
// Our $ context function gets a value from context.
// We clone the context to avoid mutation in the binding affecting real
// app context.
const clonedContext = cloneDeep({ ...context, snippets: null })
let clonedContext: Record<string, any>
if (isBackendService()) {
// On the backned, values are copied across the isolated-vm boundary and
// so we don't need to do any cloning here. This does create a fundamental
// difference in how JS executes on the frontend vs the backend, e.g.
// consider this snippet:
//
// $("array").push(2)
// return $("array")[1]
//
// With the context of `{ array: [1] }`, the backend will return
// `undefined` whereas the frontend will return `2`. We should fix this.
clonedContext = context
} else {
clonedContext = cloneDeep(context)
}
const sandboxContext = {
$: (path: string) => getContextValue(path, clonedContext),
helpers: getJsHelperList(),

View File

@ -1,4 +1,4 @@
import { Context, createContext, runInNewContext } from "vm"
import { createContext, runInNewContext } from "vm"
import { create, TemplateDelegate } from "handlebars"
import { registerAll, registerMinimum } from "./helpers/index"
import { postprocess, preprocess } from "./processors"
@ -455,21 +455,14 @@ export function convertToJS(hbs: string) {
export { JsTimeoutError, UserScriptError } from "./errors"
export function defaultJSSetup() {
if (!isBackendService()) {
/**
* Use polyfilled vm to run JS scripts in a browser Env
*/
setJSRunner((js: string, context: Context) => {
context = {
...context,
alert: undefined,
setInterval: undefined,
setTimeout: undefined,
}
createContext(context)
export function browserJSSetup() {
/**
* Use polyfilled vm to run JS scripts in a browser Env
*/
setJSRunner((js: string, context: Record<string, any>) => {
createContext(context)
const wrappedJs = `
const wrappedJs = `
result = {
result: null,
error: null,
@ -484,12 +477,17 @@ export function defaultJSSetup() {
result;
`
const result = runInNewContext(wrappedJs, context, { timeout: 1000 })
if (result.error) {
throw new UserScriptError(result.error)
}
return result.result
})
const result = runInNewContext(wrappedJs, context, { timeout: 1000 })
if (result.error) {
throw new UserScriptError(result.error)
}
return result.result
})
}
export function defaultJSSetup() {
if (!isBackendService()) {
browserJSSetup()
} else {
removeJSRunner()
}

View File

@ -4,7 +4,14 @@ export const FIND_HBS_REGEX = /{{([^{].*?)}}/g
export const FIND_ANY_HBS_REGEX = /{?{{([^{].*?)}}}?/g
export const FIND_TRIPLE_HBS_REGEX = /{{{([^{].*?)}}}/g
const isJest = () => typeof jest !== "undefined"
export const isBackendService = () => {
// We consider the tests for string-templates to be frontend, so that they
// test the frontend JS functionality.
if (isJest()) {
return false
}
return typeof window === "undefined"
}

View File

@ -1,7 +1,13 @@
import vm from "vm"
import { processStringSync, encodeJSBinding, setJSRunner } from "../src/index"
import {
processStringSync,
encodeJSBinding,
defaultJSSetup,
} from "../src/index"
import { UUID_REGEX } from "./constants"
import tk from "timekeeper"
const DATE = "2021-01-21T12:00:00"
tk.freeze(DATE)
const processJS = (js: string, context?: object): any => {
return processStringSync(encodeJSBinding(js), context)
@ -9,9 +15,7 @@ const processJS = (js: string, context?: object): any => {
describe("Javascript", () => {
beforeAll(() => {
setJSRunner((js, context) => {
return vm.runInNewContext(js, context, { timeout: 1000 })
})
defaultJSSetup()
})
describe("Test the JavaScript helper", () => {
@ -118,8 +122,7 @@ describe("Javascript", () => {
})
it("should handle errors", () => {
const output = processJS(`throw "Error"`)
expect(output).toBe("Error while executing JS")
expect(processJS(`throw "Error"`)).toEqual("Error")
})
it("should timeout after one second", () => {
@ -127,16 +130,18 @@ describe("Javascript", () => {
expect(output).toBe("Timed out while executing JS")
})
it("should prevent access to the process global", () => {
const output = processJS(`return process`)
expect(output).toBe("Error while executing JS")
it("should prevent access to the process global", async () => {
expect(processJS(`return process`)).toEqual(
"ReferenceError: process is not defined"
)
})
})
describe("check JS helpers", () => {
it("should error if using the format helper. not helpers.", () => {
const output = processJS(`return helper.toInt(4.3)`)
expect(output).toBe("Error while executing JS")
expect(processJS(`return helper.toInt(4.3)`)).toEqual(
"ReferenceError: helper is not defined"
)
})
it("should be able to use toInt", () => {
@ -156,4 +161,323 @@ describe("Javascript", () => {
expect(output).toBe("Custom")
})
})
describe("mutability", () => {
it("should not allow the context to be mutated", async () => {
const context = { array: [1] }
const result = await processJS(
`
const array = $("array");
array.push(2);
return array[1]
`,
context
)
expect(result).toEqual(2)
expect(context.array).toEqual([1])
})
})
describe("malice", () => {
it("should not be able to call JS functions", () => {
expect(processJS(`return alert("hello")`)).toEqual(
"ReferenceError: alert is not defined"
)
expect(processJS(`return prompt("hello")`)).toEqual(
"ReferenceError: prompt is not defined"
)
expect(processJS(`return confirm("hello")`)).toEqual(
"ReferenceError: confirm is not defined"
)
expect(processJS(`return setTimeout(() => {}, 1000)`)).toEqual(
"ReferenceError: setTimeout is not defined"
)
expect(processJS(`return setInterval(() => {}, 1000)`)).toEqual(
"ReferenceError: setInterval is not defined"
)
})
})
// the test cases here were extracted from templates/real world examples of JS in Budibase
describe("real test cases from Budicloud", () => {
const context = {
"Unit Value": 2,
Quantity: 1,
}
it("handle test case 1", async () => {
const result = await processJS(
`
var Gross = $("[Unit Value]") * $("[Quantity]")
return Gross.toFixed(2)`,
context
)
expect(result).toBeDefined()
expect(result).toBe("2.00")
})
it("handle test case 2", async () => {
const todayDate = new Date()
// add a year and a month
todayDate.setMonth(new Date().getMonth() + 1)
todayDate.setFullYear(todayDate.getFullYear() + 1)
const context = {
"Purchase Date": DATE,
today: todayDate.toISOString(),
}
const result = await processJS(
`
var purchase = new Date($("[Purchase Date]"));
let purchaseyear = purchase.getFullYear();
let purchasemonth = purchase.getMonth();
var today = new Date($("today"));
let todayyear = today.getFullYear();
let todaymonth = today.getMonth();
var age = todayyear - purchaseyear
if (((todaymonth - purchasemonth) < 6) == true){
return age
}
`,
context
)
expect(result).toBeDefined()
expect(result).toBe(1)
})
it("should handle test case 3", async () => {
const context = {
Escalate: true,
"Budget ($)": 1100,
}
const result = await processJS(
`
if ($("[Escalate]") == true) {
if ($("Budget ($)") <= 1000)
{return 2;}
if ($("Budget ($)") > 1000)
{return 3;}
}
else {
if ($("Budget ($)") <= 1000)
{return 1;}
if ($("Budget ($)") > 1000)
if ($("Budget ($)") < 10000)
{return 2;}
else
{return 3}
}
`,
context
)
expect(result).toBeDefined()
expect(result).toBe(3)
})
it("should handle test case 4", async () => {
const context = {
"Time Sheets": ["a", "b"],
}
const result = await processJS(
`
let hours = 0
if (($("[Time Sheets]") != null) == true){
for (i = 0; i < $("[Time Sheets]").length; i++){
let hoursLogged = "Time Sheets." + i + ".Hours"
hours += $(hoursLogged)
}
return hours
}
if (($("[Time Sheets]") != null) == false){
return hours
}
`,
context
)
expect(result).toBeDefined()
expect(result).toBe("0ab")
})
it("should handle test case 5", async () => {
const context = {
change: JSON.stringify({ a: 1, primaryDisplay: "a" }),
previous: JSON.stringify({ a: 2, primaryDisplay: "b" }),
}
const result = await processJS(
`
let change = $("[change]") ? JSON.parse($("[change]")) : {}
let previous = $("[previous]") ? JSON.parse($("[previous]")) : {}
function simplifyLink(originalKey, value, parent) {
if (Array.isArray(value)) {
if (value.filter(item => Object.keys(item || {}).includes("primaryDisplay")).length > 0) {
parent[originalKey] = value.map(link => link.primaryDisplay)
}
}
}
for (let entry of Object.entries(change)) {
simplifyLink(entry[0], entry[1], change)
}
for (let entry of Object.entries(previous)) {
simplifyLink(entry[0], entry[1], previous)
}
let diff = Object.fromEntries(Object.entries(change).filter(([k, v]) => previous[k]?.toString() !== v?.toString()))
delete diff.audit_change
delete diff.audit_previous
delete diff._id
delete diff._rev
delete diff.tableId
delete diff.audit
for (let entry of Object.entries(diff)) {
simplifyLink(entry[0], entry[1], diff)
}
return JSON.stringify(change)?.replaceAll(",\\"", ",\\n\\t\\"").replaceAll("{\\"", "{\\n\\t\\"").replaceAll("}", "\\n}")
`,
context
)
expect(result).toBe(`{\n\t"a":1,\n\t"primaryDisplay":"a"\n}`)
})
it("should handle test case 6", async () => {
const context = {
"Join Date": DATE,
}
const result = await processJS(
`
var rate = 5;
var today = new Date();
// comment
function monthDiff(dateFrom, dateTo) {
return dateTo.getMonth() - dateFrom.getMonth() +
(12 * (dateTo.getFullYear() - dateFrom.getFullYear()))
}
var serviceMonths = monthDiff( new Date($("[Join Date]")), today);
var serviceYears = serviceMonths / 12;
if (serviceYears >= 1 && serviceYears < 5){
rate = 10;
}
if (serviceYears >= 5 && serviceYears < 10){
rate = 15;
}
if (serviceYears >= 10){
rate = 15;
rate += 0.5 * (Number(serviceYears.toFixed(0)) - 10);
}
return rate;
`,
context
)
expect(result).toBe(10)
})
it("should handle test case 7", async () => {
const context = {
"P I": "Pass",
"PA I": "Pass",
"F I": "Fail",
"V I": "Pass",
}
const result = await processJS(
`if (($("[P I]") == "Pass") == true)
if (($("[ P I]") == "Pass") == true)
if (($("[F I]") == "Pass") == true)
if (($("[V I]") == "Pass") == true)
{return "Pass"}
if (($("[PA I]") == "Fail") == true)
{return "Fail"}
if (($("[ P I]") == "Fail") == true)
{return "Fail"}
if (($("[F I]") == "Fail") == true)
{return "Fail"}
if (($("[V I]") == "Fail") == true)
{return "Fail"}
else
{return ""}`,
context
)
expect(result).toBe("Fail")
})
it("should handle test case 8", async () => {
const context = {
"T L": [{ Hours: 10 }],
"B H": 50,
}
const result = await processJS(
`var totalHours = 0;
if (($("[T L]") != null) == true){
for (let i = 0; i < ($("[T L]").length); i++){
var individualHours = "T L." + i + ".Hours";
var hoursNum = Number($(individualHours));
totalHours += hoursNum;
}
return totalHours.toFixed(2);
}
if (($("[T L]") != null) == false) {
return totalHours.toFixed(2);
}
`,
context
)
expect(result).toBe("10.00")
})
it("should handle test case 9", async () => {
const context = {
"T L": [{ Hours: 10 }],
"B H": 50,
}
const result = await processJS(
`var totalHours = 0;
if (($("[T L]") != null) == true){
for (let i = 0; i < ($("[T L]").length); i++){
var individualHours = "T L." + i + ".Hours";
var hoursNum = Number($(individualHours));
totalHours += hoursNum;
}
return ($("[B H]") - totalHours).toFixed(2);
}
if (($("[T L]") != null) == false) {
return ($("[B H]") - totalHours).toFixed(2);
}`,
context
)
expect(result).toBe("40.00")
})
it("should handle test case 10", async () => {
const context = {
"F F": [{ "F S": 10 }],
}
const result = await processJS(
`var rating = 0;
if ($("[F F]") != null){
for (i = 0; i < $("[F F]").length; i++){
var individualRating = $("F F." + i + ".F S");
rating += individualRating;
}
rating = (rating / $("[F F]").length);
}
return rating;
`,
context
)
expect(result).toBe(10)
})
})
})

View File

@ -1,5 +1,3 @@
import vm from "vm"
jest.mock("@budibase/handlebars-helpers/lib/math", () => {
const actual = jest.requireActual("@budibase/handlebars-helpers/lib/math")
@ -17,7 +15,7 @@ jest.mock("@budibase/handlebars-helpers/lib/uuid", () => {
}
})
import { processString, setJSRunner } from "../src/index"
import { defaultJSSetup, processString } from "../src/index"
import tk from "timekeeper"
import { getParsedManifest, runJsHelpersTests } from "./utils"
@ -32,9 +30,7 @@ describe("manifest", () => {
const manifest = getParsedManifest()
beforeAll(() => {
setJSRunner((js, context) => {
return vm.runInNewContext(js, context, { timeout: 1000 })
})
defaultJSSetup()
})
describe("examples are valid", () => {

View File

@ -12,7 +12,6 @@ export interface CreateAccountRequest {
name?: string
password: string
provider?: AccountSSOProvider
thirdPartyProfile: object
}
export interface SearchAccountsRequest {

View File

@ -7,10 +7,9 @@ export interface CreateAccount {
tenantId: string
hosting: Hosting
authType: AuthType
accountName: string
// optional fields - for sso based sign ups
registrationStep?: string
// profile
tenantName?: string
name?: string
size?: string
profession?: string
@ -20,11 +19,6 @@ export interface CreatePassswordAccount extends CreateAccount {
password: string
}
export interface CreateVerifiableSSOAccount extends CreateAccount {
provider?: AccountSSOProvider
thirdPartyProfile?: any
}
export const isCreatePasswordAccount = (
account: CreateAccount
): account is CreatePassswordAccount => account.authType === AuthType.PASSWORD
@ -56,6 +50,7 @@ export interface Account extends CreateAccount {
providerType?: AccountSSOProviderType
quotaUsage?: QuotaUsage
offlineLicenseToken?: string
tenantName?: string
}
export interface PasswordAccount extends Account {
@ -103,8 +98,6 @@ export interface AccountSSO {
provider: AccountSSOProvider
providerType: AccountSSOProviderType
oauth2?: OAuthTokens
pictureUrl?: string
thirdPartyProfile: any // TODO: define what the google profile looks like
}
export type SSOAccount = (Account | CloudAccount) & AccountSSO

View File

@ -79,10 +79,15 @@ export enum CalculationType {
MAX = "max",
}
export enum ViewV2Type {
CALCULATION = "calculation",
}
export interface ViewV2 {
version: 2
id: string
name: string
type?: ViewV2Type
primaryDisplay?: string
tableId: string
query?: LegacyFilter[] | SearchFilters
@ -94,7 +99,6 @@ export interface ViewV2 {
type?: SortType
}
schema?: ViewV2Schema
uiMetadata?: Record<string, any>
}
export type ViewV2Schema = Record<string, ViewFieldMetadata>

View File

@ -21,7 +21,6 @@ export interface UserSSO {
provider: string // the individual provider e.g. Okta, Auth0, Google
providerType: SSOProviderType
oauth2?: OAuth2
thirdPartyProfile?: SSOProfileJson
profile?: {
displayName?: string
name?: {
@ -45,7 +44,6 @@ export interface User extends Document {
userId?: string
firstName?: string
lastName?: string
pictureUrl?: string
forceResetPassword?: boolean
roles: UserRoles
builder?: {