MirrorSave
/
budibase
mirror of https://github.com/Budibase/budibase.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Merge branch 'master' of github.com:Budibase/budibase into v3-ui

This commit is contained in:
Andrew Kingston 2024-10-08 14:06:26 +01:00
parent fda5fdc16e e3c6b60211
commit 76ce8b5fd5
No known key found for this signature in database
26 changed files with 794 additions and 175 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
charts/budibase/templates/app-service-deployment.yaml
View File

@ -184,6 +184,10 @@ spec:
- name: NODE_DEBUG
value: {{ .Values.services.apps.nodeDebug | quote }}
{{ end }}
{{ if .Values.services.apps.xssSafeMode }}
- name: XSS_SAFE_MODE
value: {{ .Values.services.apps.xssSafeMode | quote }}
{{ end }}
{{ if .Values.globals.datadogApmEnabled }}
- name: DD_LOGS_INJECTION
value: {{ .Values.globals.datadogApmEnabled | quote }}

2
lerna.json
View File

@ -1,6 +1,6 @@
{
"$schema": "node_modules/lerna/schemas/lerna-schema.json",
"version": "2.32.11",
"version": "2.32.12",
"npmClient": "yarn",
"packages": [
"packages/*",

2
packages/account-portal

@ -1 +1 @@
Subproject commit 3e24f6293ff5ee5f9b42822e001504e3bbf19cc0
Subproject commit 8cd052ce8288f343812a514d06c5a9459b3ba1a8

14
packages/backend-core/src/db/couch/DatabaseImpl.ts
View File

@ -371,11 +371,21 @@ export class DatabaseImpl implements Database {
return this.performCall(() => {
return async () => {
const response = await directCouchUrlCall(args)
const json = await response.json()
const text = await response.text()
if (response.status > 300) {
let json
try {
json = JSON.parse(text)
} catch (err) {
console.error(`SQS error: ${text}`)
throw new CouchDBError(
"error while running SQS query, please try again later",
{ name: "sqs_error", status: response.status }
)
}
throw json
}
return json as T
return JSON.parse(text) as T
}
})
}

24
packages/backend-core/src/middleware/passport/sso/sso.ts
View File

@ -2,7 +2,6 @@ import { generateGlobalUserID } from "../../../db"
import { authError } from "../utils"
import * as users from "../../../users"
import * as context from "../../../context"
import fetch from "node-fetch"
import {
SaveSSOUserFunction,
SSOAuthDetails,
@ -97,28 +96,13 @@ export async function authenticate(
return done(null, ssoUser)
}
async function getProfilePictureUrl(user: User, details: SSOAuthDetails) {
const pictureUrl = details.profile?._json.picture
if (pictureUrl) {
const response = await fetch(pictureUrl)
if (response.status === 200) {
const type = response.headers.get("content-type") as string
if (type.startsWith("image/")) {
return pictureUrl
}
}
}
}
/**
* @returns a user that has been sync'd with third party information
*/
async function syncUser(user: User, details: SSOAuthDetails): Promise<SSOUser> {
let firstName
let lastName
let pictureUrl
let oauth2
let thirdPartyProfile
if (details.profile) {
const profile = details.profile
@ -134,12 +118,6 @@ async function syncUser(user: User, details: SSOAuthDetails): Promise<SSOUser> {
lastName = name.familyName
}
}
pictureUrl = await getProfilePictureUrl(user, details)
thirdPartyProfile = {
...profile._json,
}
}
// oauth tokens for future use
@ -155,8 +133,6 @@ async function syncUser(user: User, details: SSOAuthDetails): Promise<SSOUser> {
providerType: details.providerType,
firstName,
lastName,
thirdPartyProfile,
pictureUrl,
oauth2,
}
}

67
packages/backend-core/tests/core/utilities/structures/accounts.ts
View File

@ -6,9 +6,6 @@ import {
AccountSSOProviderType,
AuthType,
CloudAccount,
CreateAccount,
CreatePassswordAccount,
CreateVerifiableSSOAccount,
Hosting,
SSOAccount,
} from "@budibase/types"
@ -19,6 +16,7 @@ export const account = (partial: Partial<Account> = {}): Account => {
accountId: uuid(),
tenantId: generator.word(),
email: generator.email({ domain: "example.com" }),
accountName: generator.word(),
tenantName: generator.word(),
hosting: Hosting.SELF,
createdAt: Date.now(),
@ -61,10 +59,8 @@ export function ssoAccount(account: Account = cloudAccount()): SSOAccount {
accessToken: generator.string(),
refreshToken: generator.string(),
},
pictureUrl: generator.url(),
provider: provider(),
providerType: providerType(),
thirdPartyProfile: {},
}
}
@ -78,68 +74,7 @@ export function verifiableSsoAccount(
accessToken: generator.string(),
refreshToken: generator.string(),
},
pictureUrl: generator.url(),
provider: AccountSSOProvider.MICROSOFT,
providerType: AccountSSOProviderType.MICROSOFT,
thirdPartyProfile: { id: "abc123" },
}
}
export const cloudCreateAccount: CreatePassswordAccount = {
email: "cloud@budibase.com",
tenantId: "cloud",
hosting: Hosting.CLOUD,
authType: AuthType.PASSWORD,
password: "Password123!",
tenantName: "cloud",
name: "Budi Armstrong",
size: "10+",
profession: "Software Engineer",
}
export const cloudSSOCreateAccount: CreateAccount = {
email: "cloud-sso@budibase.com",
tenantId: "cloud-sso",
hosting: Hosting.CLOUD,
authType: AuthType.SSO,
tenantName: "cloudsso",
name: "Budi Armstrong",
size: "10+",
profession: "Software Engineer",
}
export const cloudVerifiableSSOCreateAccount: CreateVerifiableSSOAccount = {
email: "cloud-sso@budibase.com",
tenantId: "cloud-sso",
hosting: Hosting.CLOUD,
authType: AuthType.SSO,
tenantName: "cloudsso",
name: "Budi Armstrong",
size: "10+",
profession: "Software Engineer",
provider: AccountSSOProvider.MICROSOFT,
thirdPartyProfile: { id: "abc123" },
}
export const selfCreateAccount: CreatePassswordAccount = {
email: "self@budibase.com",
tenantId: "self",
hosting: Hosting.SELF,
authType: AuthType.PASSWORD,
password: "Password123!",
tenantName: "self",
name: "Budi Armstrong",
size: "10+",
profession: "Software Engineer",
}
export const selfSSOCreateAccount: CreateAccount = {
email: "self-sso@budibase.com",
tenantId: "self-sso",
hosting: Hosting.SELF,
authType: AuthType.SSO,
tenantName: "selfsso",
name: "Budi Armstrong",
size: "10+",
profession: "Software Engineer",
}

5
packages/backend-core/tests/core/utilities/structures/users.ts
View File

@ -25,7 +25,6 @@ export const user = (userProps?: Partial<Omit<User, "userId">>): User => {
roles: { app_test: "admin" },
firstName: generator.first(),
lastName: generator.last(),
pictureUrl: "http://example.com",
tenantId: tenant.id(),
...userProps,
}
@ -86,9 +85,5 @@ export function ssoUser(
oauth2: opts.details?.oauth2,
provider: opts.details?.provider!,
providerType: opts.details?.providerType!,
thirdPartyProfile: {
email: base.email,
picture: base.pictureUrl,
},
}
}

4
packages/server/src/api/controllers/view/viewsV2.ts
View File

@ -127,13 +127,13 @@ export async function create(ctx: Ctx<CreateViewRequest, ViewResponse>) {
const parsedView: Omit<RequiredKeys<ViewV2>, "id" | "version"> = {
name: view.name,
type: view.type,
tableId: view.tableId,
query: view.query,
queryUI: view.queryUI,
sort: view.sort,
schema,
primaryDisplay: view.primaryDisplay,
uiMetadata: view.uiMetadata,
}
const result = await sdk.views.create(tableId, parsedView)
ctx.status = 201
@ -163,6 +163,7 @@ export async function update(ctx: Ctx<UpdateViewRequest, ViewResponse>) {
const parsedView: RequiredKeys<ViewV2> = {
id: view.id,
name: view.name,
type: view.type,
version: view.version,
tableId: view.tableId,
query: view.query,
@ -170,7 +171,6 @@ export async function update(ctx: Ctx<UpdateViewRequest, ViewResponse>) {
sort: view.sort,
schema,
primaryDisplay: view.primaryDisplay,
uiMetadata: view.uiMetadata,
}
const result = await sdk.views.update(tableId, parsedView)

245
packages/server/src/api/routes/tests/viewV2.spec.ts
View File

@ -26,6 +26,7 @@ import {
BBReferenceFieldSubType,
NumericCalculationFieldMetadata,
ViewV2Schema,
ViewV2Type,
} from "@budibase/types"
import { generator, mocks } from "@budibase/backend-core/tests"
import { DatabaseName, getDatasource } from "../../../integrations/tests/utils"
@ -155,7 +156,7 @@ describe.each([
})
it("can persist views with all fields", async () => {
const newView: Required<Omit<CreateViewRequest, "queryUI">> = {
const newView: Required<Omit<CreateViewRequest, "queryUI" | "type">> = {
name: generator.name(),
tableId: table._id!,
primaryDisplay: "id",
@ -177,9 +178,6 @@ describe.each([
visible: true,
},
},
uiMetadata: {
foo: "bar",
},
}
const res = await config.api.viewV2.create(newView)
@ -549,6 +547,7 @@ describe.each([
let view = await config.api.viewV2.create({
tableId: table._id!,
name: generator.guid(),
type: ViewV2Type.CALCULATION,
schema: {
sum: {
visible: true,
@ -571,6 +570,184 @@ describe.each([
expect(sum.calculationType).toEqual(CalculationType.SUM)
expect(sum.field).toEqual("Price")
})
it("cannot create a view with calculation fields unless it has the right type", async () => {
await config.api.viewV2.create(
{
tableId: table._id!,
name: generator.guid(),
schema: {
sum: {
visible: true,
calculationType: CalculationType.SUM,
field: "Price",
},
},
},
{
status: 400,
body: {
message:
"Calculation fields are not allowed in non-calculation views",
},
}
)
})
it("cannot create a calculation view with more than 5 aggregations", async () => {
await config.api.viewV2.create(
{
tableId: table._id!,
name: generator.guid(),
type: ViewV2Type.CALCULATION,
schema: {
sum: {
visible: true,
calculationType: CalculationType.SUM,
field: "Price",
},
count: {
visible: true,
calculationType: CalculationType.COUNT,
field: "Price",
},
min: {
visible: true,
calculationType: CalculationType.MIN,
field: "Price",
},
max: {
visible: true,
calculationType: CalculationType.MAX,
field: "Price",
},
avg: {
visible: true,
calculationType: CalculationType.AVG,
field: "Price",
},
sum2: {
visible: true,
calculationType: CalculationType.SUM,
field: "Price",
},
},
},
{
status: 400,
body: {
message: "Calculation views can only have a maximum of 5 fields",
},
}
)
})
it("cannot create a calculation view with duplicate calculations", async () => {
await config.api.viewV2.create(
{
tableId: table._id!,
name: generator.guid(),
type: ViewV2Type.CALCULATION,
schema: {
sum: {
visible: true,
calculationType: CalculationType.SUM,
field: "Price",
},
sum2: {
visible: true,
calculationType: CalculationType.SUM,
field: "Price",
},
},
},
{
status: 400,
body: {
message:
'Duplicate calculation on field "Price", calculation type "sum"',
},
}
)
})
it("finds duplicate counts", async () => {
await config.api.viewV2.create(
{
tableId: table._id!,
name: generator.guid(),
type: ViewV2Type.CALCULATION,
schema: {
count: {
visible: true,
calculationType: CalculationType.COUNT,
},
count2: {
visible: true,
calculationType: CalculationType.COUNT,
},
},
},
{
status: 400,
body: {
message:
'Duplicate calculation on field "*", calculation type "count"',
},
}
)
})
it("finds duplicate count distincts", async () => {
await config.api.viewV2.create(
{
tableId: table._id!,
name: generator.guid(),
type: ViewV2Type.CALCULATION,
schema: {
count: {
visible: true,
calculationType: CalculationType.COUNT,
distinct: true,
field: "Price",
},
count2: {
visible: true,
calculationType: CalculationType.COUNT,
distinct: true,
field: "Price",
},
},
},
{
status: 400,
body: {
message:
'Duplicate calculation on field "Price", calculation type "count"',
},
}
)
})
it("does not confuse counts and count distincts in the duplicate check", async () => {
await config.api.viewV2.create({
tableId: table._id!,
name: generator.guid(),
type: ViewV2Type.CALCULATION,
schema: {
count: {
visible: true,
calculationType: CalculationType.COUNT,
},
count2: {
visible: true,
calculationType: CalculationType.COUNT,
distinct: true,
field: "Price",
},
},
})
})
})
describe("update", () => {
@ -615,7 +792,9 @@ describe.each([
it("can update all fields", async () => {
const tableId = table._id!
const updatedData: Required<Omit<UpdateViewRequest, "queryUI">> = {
const updatedData: Required<
Omit<UpdateViewRequest, "queryUI" | "type">
> = {
version: view.version,
id: view.id,
tableId,
@ -643,9 +822,6 @@ describe.each([
readonly: true,
},
},
uiMetadata: {
foo: "bar",
},
}
await config.api.viewV2.update(updatedData)
@ -830,6 +1006,32 @@ describe.each([
)
})
it("cannot update view type after creation", async () => {
const view = await config.api.viewV2.create({
tableId: table._id!,
name: generator.guid(),
schema: {
id: { visible: true },
Price: {
visible: true,
},
},
})
await config.api.viewV2.update(
{
...view,
type: ViewV2Type.CALCULATION,
},
{
status: 400,
body: {
message: "Cannot update view type after creation",
},
}
)
})
isInternal &&
it("updating schema will only validate modified field", async () => {
let view = await config.api.viewV2.create({
@ -902,6 +1104,7 @@ describe.each([
view = await config.api.viewV2.create({
tableId: table._id!,
name: generator.guid(),
type: ViewV2Type.CALCULATION,
schema: {
country: {
visible: true,
@ -1024,6 +1227,26 @@ describe.each([
)
})
it("can add a new group by field that is invisible, even if required on the table", async () => {
view.schema!.name = { visible: false }
await config.api.viewV2.update(view)
const { rows } = await config.api.row.search(view.id)
expect(rows).toHaveLength(2)
expect(rows).toEqual(
expect.arrayContaining([
{
country: "USA",
age: 65,
},
{
country: "UK",
age: 61,
},
])
)
})
it("can add a new calculation field", async () => {
view.schema!.count = {
visible: true,
@ -2639,6 +2862,7 @@ describe.each([
it("should be able to search by calculations", async () => {
const view = await config.api.viewV2.create({
tableId: table._id!,
type: ViewV2Type.CALCULATION,
name: generator.guid(),
schema: {
"Quantity Sum": {
@ -2673,6 +2897,7 @@ describe.each([
const view = await config.api.viewV2.create({
tableId: table._id!,
name: generator.guid(),
type: ViewV2Type.CALCULATION,
schema: {
quantity: {
visible: true,
@ -2711,6 +2936,7 @@ describe.each([
const view = await config.api.viewV2.create({
tableId: table._id!,
name: generator.guid(),
type: ViewV2Type.CALCULATION,
schema: {
aggregate: {
visible: true,
@ -2771,6 +2997,7 @@ describe.each([
const view = await config.api.viewV2.create({
tableId: table._id!,
name: generator.guid(),
type: ViewV2Type.CALCULATION,
schema: {
count: {
visible: true,
@ -2805,6 +3032,7 @@ describe.each([
{
tableId: table._id!,
name: generator.guid(),
type: ViewV2Type.CALCULATION,
schema: {
count: {
visible: true,
@ -2853,6 +3081,7 @@ describe.each([
const view = await config.api.viewV2.create({
tableId: table._id!,
name: generator.guid(),
type: ViewV2Type.CALCULATION,
schema: {
sum: {
visible: true,

1
packages/server/src/environment.ts
View File

@ -83,6 +83,7 @@ const environment = {
PLUGINS_DIR: process.env.PLUGINS_DIR || DEFAULTS.PLUGINS_DIR,
MAX_IMPORT_SIZE_MB: process.env.MAX_IMPORT_SIZE_MB,
SESSION_EXPIRY_SECONDS: process.env.SESSION_EXPIRY_SECONDS,
XSS_SAFE_MODE: process.env.XSS_SAFE_MODE,
// SQL
SQL_MAX_ROWS: process.env.SQL_MAX_ROWS,
SQL_LOGGING_ENABLE: process.env.SQL_LOGGING_ENABLE,

28
packages/server/src/jsRunner/tests/jsRunner.spec.ts
View File

@ -8,7 +8,6 @@ import { init } from ".."
import TestConfiguration from "../../tests/utilities/TestConfiguration"
const DATE = "2021-01-21T12:00:00"
tk.freeze(DATE)
describe("jsRunner (using isolated-vm)", () => {
@ -48,6 +47,33 @@ describe("jsRunner (using isolated-vm)", () => {
).toEqual("ReferenceError: process is not defined")
})
it("should not allow the context to be mutated", async () => {
const context = { array: [1] }
const result = await processJS(
`
const array = $("array");
array.push(2);
return array[1]
`,
context
)
expect(result).toEqual(2)
expect(context.array).toEqual([1])
})
it("should copy values whenever returning them from $", async () => {
const context = { array: [1] }
const result = await processJS(
`
$("array").push(2);
return $("array")[1];
`,
context
)
expect(result).toEqual(undefined)
expect(context.array).toEqual([1])
})
describe("helpers", () => {
runJsHelpersTests({
funcWrap: (func: any) => config.doInContext(config.getAppId(), func),

43
packages/server/src/sdk/app/rows/tests/utils.spec.ts
View File

@ -8,6 +8,7 @@ import {
import { generateTableID } from "../../../../db/utils"
import { validate } from "../utils"
import { generator } from "@budibase/backend-core/tests"
import { withEnv } from "../../../../environment"
describe("validate", () => {
const hour = () => generator.hour().toString().padStart(2, "0")
@ -332,4 +333,46 @@ describe("validate", () => {
})
})
})
describe("XSS Safe mode", () => {
const getTable = (): Table => ({
type: "table",
_id: generateTableID(),
name: "table",
sourceId: INTERNAL_TABLE_SOURCE_ID,
sourceType: TableSourceType.INTERNAL,
schema: {
text: {
name: "sometext",
type: FieldType.STRING,
},
},
})
it.each([
"SELECT * FROM users WHERE username = 'admin' --",
"SELECT * FROM users WHERE id = 1; DROP TABLE users;",
"1' OR '1' = '1",
"' OR 'a' = 'a",
"<script>alert('XSS');</script>",
'"><img src=x onerror=alert(1)>',
"</script><script>alert('test')</script>",
"<div onmouseover=\"alert('XSS')\">Hover over me!</div>",
"'; EXEC sp_msforeachtable 'DROP TABLE ?'; --",
"{alert('Injected')}",
"UNION SELECT * FROM users",
"INSERT INTO users (username, password) VALUES ('admin', 'password')",
"/* This is a comment */ SELECT * FROM users",
'<iframe src="http://malicious-site.com"></iframe>',
])("test potentially unsafe input: %s", async input => {
await withEnv({ XSS_SAFE_MODE: "1" }, async () => {
const table = getTable()
const row = { text: input }
const output = await validate({ source: table, row })
expect(output.valid).toBe(false)
expect(output.errors).toStrictEqual({
text: ["Input not sanitised - potentially vulnerable to XSS"],
})
})
})
})
})

13
packages/server/src/sdk/app/rows/utils.ts
View File

@ -22,6 +22,7 @@ import { extractViewInfoFromID, isRelationshipColumn } from "../../../db/utils"
import { isSQL } from "../../../integrations/utils"
import { docIds, sql } from "@budibase/backend-core"
import { getTableFromSource } from "../../../api/controllers/row/utils"
import env from "../../../environment"
const SQL_CLIENT_SOURCE_MAP: Record<SourceName, SqlClient | undefined> = {
[SourceName.POSTGRES]: SqlClient.POSTGRES,
@ -43,6 +44,9 @@ const SQL_CLIENT_SOURCE_MAP: Record<SourceName, SqlClient | undefined> = {
[SourceName.BUDIBASE]: undefined,
}
const XSS_INPUT_REGEX =
/[<>;"'(){}]|--|\/\*|\*\/|union|select|insert|drop|delete|update|exec|script/i
export function getSQLClient(datasource: Datasource): SqlClient {
if (!isSQL(datasource)) {
throw new Error("Cannot get SQL Client for non-SQL datasource")
@ -222,6 +226,15 @@ export async function validate({
} else {
res = validateJs.single(row[fieldName], constraints)
}
if (env.XSS_SAFE_MODE && typeof row[fieldName] === "string") {
if (XSS_INPUT_REGEX.test(row[fieldName])) {
errors[fieldName] = [
"Input not sanitised - potentially vulnerable to XSS",
]
}
}
if (res) errors[fieldName] = res
}
return { valid: Object.keys(errors).length === 0, errors }

3
packages/server/src/sdk/app/views/external.ts
View File

@ -70,6 +70,9 @@ export async function update(tableId: string, view: ViewV2): Promise<ViewV2> {
if (!existingView || !existingView.name) {
throw new HTTPError(`View ${view.id} not found in table ${tableId}`, 404)
}
if (isV2(existingView) && existingView.type !== view.type) {
throw new HTTPError(`Cannot update view type after creation`, 400)
}
delete views[existingView.name]
views[view.name] = view

41
packages/server/src/sdk/app/views/index.ts
View File

@ -62,11 +62,31 @@ async function guardCalculationViewSchema(
view: Omit<ViewV2, "id" | "version">
) {
const calculationFields = helpers.views.calculationFields(view)
for (const calculationFieldName of Object.keys(calculationFields)) {
const schema = calculationFields[calculationFieldName]
if (Object.keys(calculationFields).length > 5) {
throw new HTTPError(
"Calculation views can only have a maximum of 5 fields",
400
)
}
const seen: Record<string, Record<CalculationType, boolean>> = {}
for (const name of Object.keys(calculationFields)) {
const schema = calculationFields[name]
const isCount = schema.calculationType === CalculationType.COUNT
const isDistinct = isCount && "distinct" in schema && schema.distinct
const field = isCount && !isDistinct ? "*" : schema.field
if (seen[field]?.[schema.calculationType]) {
throw new HTTPError(
`Duplicate calculation on field "${field}", calculation type "${schema.calculationType}"`,
400
)
}
seen[field] ??= {} as Record<CalculationType, boolean>
seen[field][schema.calculationType] = true
// Count fields that aren't distinct don't need to reference another field,
// so we don't validate it.
if (isCount && !isDistinct) {
@ -76,14 +96,14 @@ async function guardCalculationViewSchema(
const targetSchema = table.schema[schema.field]
if (!targetSchema) {
throw new HTTPError(
`Calculation field "${calculationFieldName}" references field "${schema.field}" which does not exist in the table schema`,
`Calculation field "${name}" references field "${schema.field}" which does not exist in the table schema`,
400
)
}
if (!isCount && !helpers.schema.isNumeric(targetSchema)) {
throw new HTTPError(
`Calculation field "${calculationFieldName}" references field "${schema.field}" which is not a numeric field`,
`Calculation field "${name}" references field "${schema.field}" which is not a numeric field`,
400
)
}
@ -109,10 +129,21 @@ async function guardViewSchema(
if (helpers.views.isCalculationView(view)) {
await guardCalculationViewSchema(table, view)
} else {
if (helpers.views.hasCalculationFields(view)) {
throw new HTTPError(
"Calculation fields are not allowed in non-calculation views",
400
)
}
}
await checkReadonlyFields(table, view)
checkRequiredFields(table, view)
if (!helpers.views.isCalculationView(view)) {
checkRequiredFields(table, view)
}
checkDisplayField(view)
}

4
packages/server/src/sdk/app/views/internal.ts
View File

@ -59,6 +59,10 @@ export async function update(tableId: string, view: ViewV2): Promise<ViewV2> {
throw new HTTPError(`View ${view.id} not found in table ${tableId}`, 404)
}
if (isV2(existingView) && existingView.type !== view.type) {
throw new HTTPError(`Cannot update view type after creation`, 400)
}
delete table.views[existingView.name]
table.views[view.name] = view
await db.put(table)

5
packages/shared-core/src/helpers/views.ts
View File

@ -3,6 +3,7 @@ import {
ViewCalculationFieldMetadata,
ViewFieldMetadata,
ViewV2,
ViewV2Type,
} from "@budibase/types"
import { pickBy } from "lodash"
@ -21,6 +22,10 @@ export function isBasicViewField(
type UnsavedViewV2 = Omit<ViewV2, "id" | "version">
export function isCalculationView(view: UnsavedViewV2) {
return view.type === ViewV2Type.CALCULATION
}
export function hasCalculationFields(view: UnsavedViewV2) {
return Object.values(view.schema || {}).some(isCalculationField)
}

39
packages/string-templates/src/helpers/javascript.ts
View File

@ -1,13 +1,14 @@
import { atob, isJSAllowed } from "../utilities"
import cloneDeep from "lodash/fp/cloneDeep"
import { atob, isBackendService, isJSAllowed } from "../utilities"
import { LITERAL_MARKER } from "../helpers/constants"
import { getJsHelperList } from "./list"
import { iifeWrapper } from "../iife"
import { JsTimeoutError, UserScriptError } from "../errors"
import { cloneDeep } from "lodash/fp"
// The method of executing JS scripts depends on the bundle being built.
// This setter is used in the entrypoint (either index.js or index.mjs).
let runJS: ((js: string, context: any) => any) | undefined = undefined
let runJS: ((js: string, context: Record<string, any>) => any) | undefined =
undefined
export const setJSRunner = (runner: typeof runJS) => (runJS = runner)
export const removeJSRunner = () => {
@ -31,9 +32,19 @@ const removeSquareBrackets = (value: string) => {
return value
}
const isReservedKey = (key: string) =>
key === "snippets" ||
key === "helpers" ||
key.startsWith("snippets.") ||
key.startsWith("helpers.")
// Our context getter function provided to JS code as $.
// Extracts a value from context.
const getContextValue = (path: string, context: any) => {
// We populate `snippets` ourselves, don't allow access to it.
if (isReservedKey(path)) {
return undefined
}
const literalStringRegex = /^(["'`]).*\1$/
let data = context
// check if it's a literal string - just return path if its quoted
@ -46,6 +57,7 @@ const getContextValue = (path: string, context: any) => {
}
data = data[removeSquareBrackets(key)]
})
return data
}
@ -67,10 +79,23 @@ export function processJS(handlebars: string, context: any) {
snippetMap[snippet.name] = snippet.code
}
// Our $ context function gets a value from context.
// We clone the context to avoid mutation in the binding affecting real
// app context.
const clonedContext = cloneDeep({ ...context, snippets: null })
let clonedContext: Record<string, any>
if (isBackendService()) {
// On the backned, values are copied across the isolated-vm boundary and
// so we don't need to do any cloning here. This does create a fundamental
// difference in how JS executes on the frontend vs the backend, e.g.
// consider this snippet:
//
// $("array").push(2)
// return $("array")[1]
//
// With the context of `{ array: [1] }`, the backend will return
// `undefined` whereas the frontend will return `2`. We should fix this.
clonedContext = context
} else {
clonedContext = cloneDeep(context)
}
const sandboxContext = {
$: (path: string) => getContextValue(path, clonedContext),
helpers: getJsHelperList(),

40
packages/string-templates/src/index.ts
View File

@ -1,4 +1,4 @@
import { Context, createContext, runInNewContext } from "vm"
import { createContext, runInNewContext } from "vm"
import { create, TemplateDelegate } from "handlebars"
import { registerAll, registerMinimum } from "./helpers/index"
import { postprocess, preprocess } from "./processors"
@ -455,21 +455,14 @@ export function convertToJS(hbs: string) {
export { JsTimeoutError, UserScriptError } from "./errors"
export function defaultJSSetup() {
if (!isBackendService()) {
/**
* Use polyfilled vm to run JS scripts in a browser Env
*/
setJSRunner((js: string, context: Context) => {
context = {
...context,
alert: undefined,
setInterval: undefined,
setTimeout: undefined,
}
createContext(context)
export function browserJSSetup() {
/**
* Use polyfilled vm to run JS scripts in a browser Env
*/
setJSRunner((js: string, context: Record<string, any>) => {
createContext(context)
const wrappedJs = `
const wrappedJs = `
result = {
result: null,
error: null,
@ -484,12 +477,17 @@ export function defaultJSSetup() {
result;
`
const result = runInNewContext(wrappedJs, context, { timeout: 1000 })
if (result.error) {
throw new UserScriptError(result.error)
}
return result.result
})
const result = runInNewContext(wrappedJs, context, { timeout: 1000 })
if (result.error) {
throw new UserScriptError(result.error)
}
return result.result
})
}
export function defaultJSSetup() {
if (!isBackendService()) {
browserJSSetup()
} else {
removeJSRunner()
}

7
packages/string-templates/src/utilities.ts
View File

@ -4,7 +4,14 @@ export const FIND_HBS_REGEX = /{{([^{].*?)}}/g
export const FIND_ANY_HBS_REGEX = /{?{{([^{].*?)}}}?/g
export const FIND_TRIPLE_HBS_REGEX = /{{{([^{].*?)}}}/g
const isJest = () => typeof jest !== "undefined"
export const isBackendService = () => {
// We consider the tests for string-templates to be frontend, so that they
// test the frontend JS functionality.
if (isJest()) {
return false
}
return typeof window === "undefined"
}

350
packages/string-templates/test/javascript.spec.ts
View File

@ -1,7 +1,13 @@
import vm from "vm"
import { processStringSync, encodeJSBinding, setJSRunner } from "../src/index"
import {
processStringSync,
encodeJSBinding,
defaultJSSetup,
} from "../src/index"
import { UUID_REGEX } from "./constants"
import tk from "timekeeper"
const DATE = "2021-01-21T12:00:00"
tk.freeze(DATE)
const processJS = (js: string, context?: object): any => {
return processStringSync(encodeJSBinding(js), context)
@ -9,9 +15,7 @@ const processJS = (js: string, context?: object): any => {
describe("Javascript", () => {
beforeAll(() => {
setJSRunner((js, context) => {
return vm.runInNewContext(js, context, { timeout: 1000 })
})
defaultJSSetup()
})
describe("Test the JavaScript helper", () => {
@ -118,8 +122,7 @@ describe("Javascript", () => {
})
it("should handle errors", () => {
const output = processJS(`throw "Error"`)
expect(output).toBe("Error while executing JS")
expect(processJS(`throw "Error"`)).toEqual("Error")
})
it("should timeout after one second", () => {
@ -127,16 +130,18 @@ describe("Javascript", () => {
expect(output).toBe("Timed out while executing JS")
})
it("should prevent access to the process global", () => {
const output = processJS(`return process`)
expect(output).toBe("Error while executing JS")
it("should prevent access to the process global", async () => {
expect(processJS(`return process`)).toEqual(
"ReferenceError: process is not defined"
)
})
})
describe("check JS helpers", () => {
it("should error if using the format helper. not helpers.", () => {
const output = processJS(`return helper.toInt(4.3)`)
expect(output).toBe("Error while executing JS")
expect(processJS(`return helper.toInt(4.3)`)).toEqual(
"ReferenceError: helper is not defined"
)
})
it("should be able to use toInt", () => {
@ -156,4 +161,323 @@ describe("Javascript", () => {
expect(output).toBe("Custom")
})
})
describe("mutability", () => {
it("should not allow the context to be mutated", async () => {
const context = { array: [1] }
const result = await processJS(
`
const array = $("array");
array.push(2);
return array[1]
`,
context
)
expect(result).toEqual(2)
expect(context.array).toEqual([1])
})
})
describe("malice", () => {
it("should not be able to call JS functions", () => {
expect(processJS(`return alert("hello")`)).toEqual(
"ReferenceError: alert is not defined"
)
expect(processJS(`return prompt("hello")`)).toEqual(
"ReferenceError: prompt is not defined"
)
expect(processJS(`return confirm("hello")`)).toEqual(
"ReferenceError: confirm is not defined"
)
expect(processJS(`return setTimeout(() => {}, 1000)`)).toEqual(
"ReferenceError: setTimeout is not defined"
)
expect(processJS(`return setInterval(() => {}, 1000)`)).toEqual(
"ReferenceError: setInterval is not defined"
)
})
})
// the test cases here were extracted from templates/real world examples of JS in Budibase
describe("real test cases from Budicloud", () => {
const context = {
"Unit Value": 2,
Quantity: 1,
}
it("handle test case 1", async () => {
const result = await processJS(
`
var Gross = $("[Unit Value]") * $("[Quantity]")
return Gross.toFixed(2)`,
context
)
expect(result).toBeDefined()
expect(result).toBe("2.00")
})
it("handle test case 2", async () => {
const todayDate = new Date()
// add a year and a month
todayDate.setMonth(new Date().getMonth() + 1)
todayDate.setFullYear(todayDate.getFullYear() + 1)
const context = {
"Purchase Date": DATE,
today: todayDate.toISOString(),
}
const result = await processJS(
`
var purchase = new Date($("[Purchase Date]"));
let purchaseyear = purchase.getFullYear();
let purchasemonth = purchase.getMonth();
var today = new Date($("today"));
let todayyear = today.getFullYear();
let todaymonth = today.getMonth();
var age = todayyear - purchaseyear
if (((todaymonth - purchasemonth) < 6) == true){
return age
}
`,
context
)
expect(result).toBeDefined()
expect(result).toBe(1)
})
it("should handle test case 3", async () => {
const context = {
Escalate: true,
"Budget ($)": 1100,
}
const result = await processJS(
`
if ($("[Escalate]") == true) {
if ($("Budget ($)") <= 1000)
{return 2;}
if ($("Budget ($)") > 1000)
{return 3;}
}
else {
if ($("Budget ($)") <= 1000)
{return 1;}
if ($("Budget ($)") > 1000)
if ($("Budget ($)") < 10000)
{return 2;}
else
{return 3}
}
`,
context
)
expect(result).toBeDefined()
expect(result).toBe(3)
})
it("should handle test case 4", async () => {
const context = {
"Time Sheets": ["a", "b"],
}
const result = await processJS(
`
let hours = 0
if (($("[Time Sheets]") != null) == true){
for (i = 0; i < $("[Time Sheets]").length; i++){
let hoursLogged = "Time Sheets." + i + ".Hours"
hours += $(hoursLogged)
}
return hours
}
if (($("[Time Sheets]") != null) == false){
return hours
}
`,
context
)
expect(result).toBeDefined()
expect(result).toBe("0ab")
})
it("should handle test case 5", async () => {
const context = {
change: JSON.stringify({ a: 1, primaryDisplay: "a" }),
previous: JSON.stringify({ a: 2, primaryDisplay: "b" }),
}
const result = await processJS(
`
let change = $("[change]") ? JSON.parse($("[change]")) : {}
let previous = $("[previous]") ? JSON.parse($("[previous]")) : {}
function simplifyLink(originalKey, value, parent) {
if (Array.isArray(value)) {
if (value.filter(item => Object.keys(item || {}).includes("primaryDisplay")).length > 0) {
parent[originalKey] = value.map(link => link.primaryDisplay)
}
}
}
for (let entry of Object.entries(change)) {
simplifyLink(entry[0], entry[1], change)
}
for (let entry of Object.entries(previous)) {
simplifyLink(entry[0], entry[1], previous)
}
let diff = Object.fromEntries(Object.entries(change).filter(([k, v]) => previous[k]?.toString() !== v?.toString()))
delete diff.audit_change
delete diff.audit_previous
delete diff._id
delete diff._rev
delete diff.tableId
delete diff.audit
for (let entry of Object.entries(diff)) {
simplifyLink(entry[0], entry[1], diff)
}
return JSON.stringify(change)?.replaceAll(",\\"", ",\\n\\t\\"").replaceAll("{\\"", "{\\n\\t\\"").replaceAll("}", "\\n}")
`,
context
)
expect(result).toBe(`{\n\t"a":1,\n\t"primaryDisplay":"a"\n}`)
})
it("should handle test case 6", async () => {
const context = {
"Join Date": DATE,
}
const result = await processJS(
`
var rate = 5;
var today = new Date();
// comment
function monthDiff(dateFrom, dateTo) {
return dateTo.getMonth() - dateFrom.getMonth() +
(12 * (dateTo.getFullYear() - dateFrom.getFullYear()))
}
var serviceMonths = monthDiff( new Date($("[Join Date]")), today);
var serviceYears = serviceMonths / 12;
if (serviceYears >= 1 && serviceYears < 5){
rate = 10;
}
if (serviceYears >= 5 && serviceYears < 10){
rate = 15;
}
if (serviceYears >= 10){
rate = 15;
rate += 0.5 * (Number(serviceYears.toFixed(0)) - 10);
}
return rate;
`,
context
)
expect(result).toBe(10)
})
it("should handle test case 7", async () => {
const context = {
"P I": "Pass",
"PA I": "Pass",
"F I": "Fail",
"V I": "Pass",
}
const result = await processJS(
`if (($("[P I]") == "Pass") == true)
if (($("[ P I]") == "Pass") == true)
if (($("[F I]") == "Pass") == true)
if (($("[V I]") == "Pass") == true)
{return "Pass"}
if (($("[PA I]") == "Fail") == true)
{return "Fail"}
if (($("[ P I]") == "Fail") == true)
{return "Fail"}
if (($("[F I]") == "Fail") == true)
{return "Fail"}
if (($("[V I]") == "Fail") == true)
{return "Fail"}
else
{return ""}`,
context
)
expect(result).toBe("Fail")
})
it("should handle test case 8", async () => {
const context = {
"T L": [{ Hours: 10 }],
"B H": 50,
}
const result = await processJS(
`var totalHours = 0;
if (($("[T L]") != null) == true){
for (let i = 0; i < ($("[T L]").length); i++){
var individualHours = "T L." + i + ".Hours";
var hoursNum = Number($(individualHours));
totalHours += hoursNum;
}
return totalHours.toFixed(2);
}
if (($("[T L]") != null) == false) {
return totalHours.toFixed(2);
}
`,
context
)
expect(result).toBe("10.00")
})
it("should handle test case 9", async () => {
const context = {
"T L": [{ Hours: 10 }],
"B H": 50,
}
const result = await processJS(
`var totalHours = 0;
if (($("[T L]") != null) == true){
for (let i = 0; i < ($("[T L]").length); i++){
var individualHours = "T L." + i + ".Hours";
var hoursNum = Number($(individualHours));
totalHours += hoursNum;
}
return ($("[B H]") - totalHours).toFixed(2);
}
if (($("[T L]") != null) == false) {
return ($("[B H]") - totalHours).toFixed(2);
}`,
context
)
expect(result).toBe("40.00")
})
it("should handle test case 10", async () => {
const context = {
"F F": [{ "F S": 10 }],
}
const result = await processJS(
`var rating = 0;
if ($("[F F]") != null){
for (i = 0; i < $("[F F]").length; i++){
var individualRating = $("F F." + i + ".F S");
rating += individualRating;
}
rating = (rating / $("[F F]").length);
}
return rating;
`,
context
)
expect(result).toBe(10)
})
})
})

8
packages/string-templates/test/manifest.spec.ts
View File

@ -1,5 +1,3 @@
import vm from "vm"
jest.mock("@budibase/handlebars-helpers/lib/math", () => {
const actual = jest.requireActual("@budibase/handlebars-helpers/lib/math")
@ -17,7 +15,7 @@ jest.mock("@budibase/handlebars-helpers/lib/uuid", () => {
}
})
import { processString, setJSRunner } from "../src/index"
import { defaultJSSetup, processString } from "../src/index"
import tk from "timekeeper"
import { getParsedManifest, runJsHelpersTests } from "./utils"
@ -32,9 +30,7 @@ describe("manifest", () => {
const manifest = getParsedManifest()
beforeAll(() => {
setJSRunner((js, context) => {
return vm.runInNewContext(js, context, { timeout: 1000 })
})
defaultJSSetup()
})
describe("examples are valid", () => {

1
packages/types/src/api/account/accounts.ts
View File

@ -12,7 +12,6 @@ export interface CreateAccountRequest {
name?: string
password: string
provider?: AccountSSOProvider
thirdPartyProfile: object
}
export interface SearchAccountsRequest {

11
packages/types/src/documents/account/account.ts
View File

@ -7,10 +7,9 @@ export interface CreateAccount {
tenantId: string
hosting: Hosting
authType: AuthType
accountName: string
// optional fields - for sso based sign ups
registrationStep?: string
// profile
tenantName?: string
name?: string
size?: string
profession?: string
@ -20,11 +19,6 @@ export interface CreatePassswordAccount extends CreateAccount {
password: string
}
export interface CreateVerifiableSSOAccount extends CreateAccount {
provider?: AccountSSOProvider
thirdPartyProfile?: any
}
export const isCreatePasswordAccount = (
account: CreateAccount
): account is CreatePassswordAccount => account.authType === AuthType.PASSWORD
@ -56,6 +50,7 @@ export interface Account extends CreateAccount {
providerType?: AccountSSOProviderType
quotaUsage?: QuotaUsage
offlineLicenseToken?: string
tenantName?: string
}
export interface PasswordAccount extends Account {
@ -103,8 +98,6 @@ export interface AccountSSO {
provider: AccountSSOProvider
providerType: AccountSSOProviderType
oauth2?: OAuthTokens
pictureUrl?: string
thirdPartyProfile: any // TODO: define what the google profile looks like
}
export type SSOAccount = (Account | CloudAccount) & AccountSSO

6
packages/types/src/documents/app/view.ts
View File

@ -79,10 +79,15 @@ export enum CalculationType {
MAX = "max",
}
export enum ViewV2Type {
CALCULATION = "calculation",
}
export interface ViewV2 {
version: 2
id: string
name: string
type?: ViewV2Type
primaryDisplay?: string
tableId: string
query?: LegacyFilter[] | SearchFilters
@ -94,7 +99,6 @@ export interface ViewV2 {
type?: SortType
}
schema?: ViewV2Schema
uiMetadata?: Record<string, any>
}
export type ViewV2Schema = Record<string, ViewFieldMetadata>

2
packages/types/src/documents/global/user.ts
View File

@ -21,7 +21,6 @@ export interface UserSSO {
provider: string // the individual provider e.g. Okta, Auth0, Google
providerType: SSOProviderType
oauth2?: OAuth2
thirdPartyProfile?: SSOProfileJson
profile?: {
displayName?: string
name?: {
@ -45,7 +44,6 @@ export interface User extends Document {
userId?: string
firstName?: string
lastName?: string
pictureUrl?: string
forceResetPassword?: boolean
roles: UserRoles
builder?: {