diff --git a/charts/budibase/templates/app-service-deployment.yaml b/charts/budibase/templates/app-service-deployment.yaml index 2b099d01f5..4d0560312f 100644 --- a/charts/budibase/templates/app-service-deployment.yaml +++ b/charts/budibase/templates/app-service-deployment.yaml @@ -184,6 +184,10 @@ spec: - name: NODE_DEBUG value: {{ .Values.services.apps.nodeDebug | quote }} {{ end }} + {{ if .Values.services.apps.xssSafeMode }} + - name: XSS_SAFE_MODE + value: {{ .Values.services.apps.xssSafeMode | quote }} + {{ end }} {{ if .Values.globals.datadogApmEnabled }} - name: DD_LOGS_INJECTION value: {{ .Values.globals.datadogApmEnabled | quote }} diff --git a/lerna.json b/lerna.json index a4bcb56d38..55721b6b2d 100644 --- a/lerna.json +++ b/lerna.json @@ -1,6 +1,6 @@ { "$schema": "node_modules/lerna/schemas/lerna-schema.json", - "version": "2.32.11", + "version": "2.32.12", "npmClient": "yarn", "packages": [ "packages/*", diff --git a/packages/account-portal b/packages/account-portal index 3e24f6293f..8cd052ce82 160000 --- a/packages/account-portal +++ b/packages/account-portal @@ -1 +1 @@ -Subproject commit 3e24f6293ff5ee5f9b42822e001504e3bbf19cc0 +Subproject commit 8cd052ce8288f343812a514d06c5a9459b3ba1a8 diff --git a/packages/backend-core/src/db/couch/DatabaseImpl.ts b/packages/backend-core/src/db/couch/DatabaseImpl.ts index 274c1b9e93..1213dde8f5 100644 --- a/packages/backend-core/src/db/couch/DatabaseImpl.ts +++ b/packages/backend-core/src/db/couch/DatabaseImpl.ts @@ -371,11 +371,21 @@ export class DatabaseImpl implements Database { return this.performCall(() => { return async () => { const response = await directCouchUrlCall(args) - const json = await response.json() + const text = await response.text() if (response.status > 300) { + let json + try { + json = JSON.parse(text) + } catch (err) { + console.error(`SQS error: ${text}`) + throw new CouchDBError( + "error while running SQS query, please try again later", + { name: "sqs_error", status: response.status } + ) + } throw json } - return json as T + return JSON.parse(text) as T } }) } diff --git a/packages/backend-core/src/middleware/passport/sso/sso.ts b/packages/backend-core/src/middleware/passport/sso/sso.ts index ee84f03dae..8901fcc56f 100644 --- a/packages/backend-core/src/middleware/passport/sso/sso.ts +++ b/packages/backend-core/src/middleware/passport/sso/sso.ts @@ -2,7 +2,6 @@ import { generateGlobalUserID } from "../../../db" import { authError } from "../utils" import * as users from "../../../users" import * as context from "../../../context" -import fetch from "node-fetch" import { SaveSSOUserFunction, SSOAuthDetails, @@ -97,28 +96,13 @@ export async function authenticate( return done(null, ssoUser) } -async function getProfilePictureUrl(user: User, details: SSOAuthDetails) { - const pictureUrl = details.profile?._json.picture - if (pictureUrl) { - const response = await fetch(pictureUrl) - if (response.status === 200) { - const type = response.headers.get("content-type") as string - if (type.startsWith("image/")) { - return pictureUrl - } - } - } -} - /** * @returns a user that has been sync'd with third party information */ async function syncUser(user: User, details: SSOAuthDetails): Promise { let firstName let lastName - let pictureUrl let oauth2 - let thirdPartyProfile if (details.profile) { const profile = details.profile @@ -134,12 +118,6 @@ async function syncUser(user: User, details: SSOAuthDetails): Promise { lastName = name.familyName } } - - pictureUrl = await getProfilePictureUrl(user, details) - - thirdPartyProfile = { - ...profile._json, - } } // oauth tokens for future use @@ -155,8 +133,6 @@ async function syncUser(user: User, details: SSOAuthDetails): Promise { providerType: details.providerType, firstName, lastName, - thirdPartyProfile, - pictureUrl, oauth2, } } diff --git a/packages/backend-core/tests/core/utilities/structures/accounts.ts b/packages/backend-core/tests/core/utilities/structures/accounts.ts index 29453ad60a..7910f3c423 100644 --- a/packages/backend-core/tests/core/utilities/structures/accounts.ts +++ b/packages/backend-core/tests/core/utilities/structures/accounts.ts @@ -6,9 +6,6 @@ import { AccountSSOProviderType, AuthType, CloudAccount, - CreateAccount, - CreatePassswordAccount, - CreateVerifiableSSOAccount, Hosting, SSOAccount, } from "@budibase/types" @@ -19,6 +16,7 @@ export const account = (partial: Partial = {}): Account => { accountId: uuid(), tenantId: generator.word(), email: generator.email({ domain: "example.com" }), + accountName: generator.word(), tenantName: generator.word(), hosting: Hosting.SELF, createdAt: Date.now(), @@ -61,10 +59,8 @@ export function ssoAccount(account: Account = cloudAccount()): SSOAccount { accessToken: generator.string(), refreshToken: generator.string(), }, - pictureUrl: generator.url(), provider: provider(), providerType: providerType(), - thirdPartyProfile: {}, } } @@ -78,68 +74,7 @@ export function verifiableSsoAccount( accessToken: generator.string(), refreshToken: generator.string(), }, - pictureUrl: generator.url(), provider: AccountSSOProvider.MICROSOFT, providerType: AccountSSOProviderType.MICROSOFT, - thirdPartyProfile: { id: "abc123" }, } } - -export const cloudCreateAccount: CreatePassswordAccount = { - email: "cloud@budibase.com", - tenantId: "cloud", - hosting: Hosting.CLOUD, - authType: AuthType.PASSWORD, - password: "Password123!", - tenantName: "cloud", - name: "Budi Armstrong", - size: "10+", - profession: "Software Engineer", -} - -export const cloudSSOCreateAccount: CreateAccount = { - email: "cloud-sso@budibase.com", - tenantId: "cloud-sso", - hosting: Hosting.CLOUD, - authType: AuthType.SSO, - tenantName: "cloudsso", - name: "Budi Armstrong", - size: "10+", - profession: "Software Engineer", -} - -export const cloudVerifiableSSOCreateAccount: CreateVerifiableSSOAccount = { - email: "cloud-sso@budibase.com", - tenantId: "cloud-sso", - hosting: Hosting.CLOUD, - authType: AuthType.SSO, - tenantName: "cloudsso", - name: "Budi Armstrong", - size: "10+", - profession: "Software Engineer", - provider: AccountSSOProvider.MICROSOFT, - thirdPartyProfile: { id: "abc123" }, -} - -export const selfCreateAccount: CreatePassswordAccount = { - email: "self@budibase.com", - tenantId: "self", - hosting: Hosting.SELF, - authType: AuthType.PASSWORD, - password: "Password123!", - tenantName: "self", - name: "Budi Armstrong", - size: "10+", - profession: "Software Engineer", -} - -export const selfSSOCreateAccount: CreateAccount = { - email: "self-sso@budibase.com", - tenantId: "self-sso", - hosting: Hosting.SELF, - authType: AuthType.SSO, - tenantName: "selfsso", - name: "Budi Armstrong", - size: "10+", - profession: "Software Engineer", -} diff --git a/packages/backend-core/tests/core/utilities/structures/users.ts b/packages/backend-core/tests/core/utilities/structures/users.ts index 0171353e23..ffddae663b 100644 --- a/packages/backend-core/tests/core/utilities/structures/users.ts +++ b/packages/backend-core/tests/core/utilities/structures/users.ts @@ -25,7 +25,6 @@ export const user = (userProps?: Partial>): User => { roles: { app_test: "admin" }, firstName: generator.first(), lastName: generator.last(), - pictureUrl: "http://example.com", tenantId: tenant.id(), ...userProps, } @@ -86,9 +85,5 @@ export function ssoUser( oauth2: opts.details?.oauth2, provider: opts.details?.provider!, providerType: opts.details?.providerType!, - thirdPartyProfile: { - email: base.email, - picture: base.pictureUrl, - }, } } diff --git a/packages/server/src/api/controllers/view/viewsV2.ts b/packages/server/src/api/controllers/view/viewsV2.ts index c98940ed88..f864df9e9e 100644 --- a/packages/server/src/api/controllers/view/viewsV2.ts +++ b/packages/server/src/api/controllers/view/viewsV2.ts @@ -127,13 +127,13 @@ export async function create(ctx: Ctx) { const parsedView: Omit, "id" | "version"> = { name: view.name, + type: view.type, tableId: view.tableId, query: view.query, queryUI: view.queryUI, sort: view.sort, schema, primaryDisplay: view.primaryDisplay, - uiMetadata: view.uiMetadata, } const result = await sdk.views.create(tableId, parsedView) ctx.status = 201 @@ -163,6 +163,7 @@ export async function update(ctx: Ctx) { const parsedView: RequiredKeys = { id: view.id, name: view.name, + type: view.type, version: view.version, tableId: view.tableId, query: view.query, @@ -170,7 +171,6 @@ export async function update(ctx: Ctx) { sort: view.sort, schema, primaryDisplay: view.primaryDisplay, - uiMetadata: view.uiMetadata, } const result = await sdk.views.update(tableId, parsedView) diff --git a/packages/server/src/api/routes/tests/viewV2.spec.ts b/packages/server/src/api/routes/tests/viewV2.spec.ts index 5238b9b752..5d565dbdbc 100644 --- a/packages/server/src/api/routes/tests/viewV2.spec.ts +++ b/packages/server/src/api/routes/tests/viewV2.spec.ts @@ -26,6 +26,7 @@ import { BBReferenceFieldSubType, NumericCalculationFieldMetadata, ViewV2Schema, + ViewV2Type, } from "@budibase/types" import { generator, mocks } from "@budibase/backend-core/tests" import { DatabaseName, getDatasource } from "../../../integrations/tests/utils" @@ -155,7 +156,7 @@ describe.each([ }) it("can persist views with all fields", async () => { - const newView: Required> = { + const newView: Required> = { name: generator.name(), tableId: table._id!, primaryDisplay: "id", @@ -177,9 +178,6 @@ describe.each([ visible: true, }, }, - uiMetadata: { - foo: "bar", - }, } const res = await config.api.viewV2.create(newView) @@ -549,6 +547,7 @@ describe.each([ let view = await config.api.viewV2.create({ tableId: table._id!, name: generator.guid(), + type: ViewV2Type.CALCULATION, schema: { sum: { visible: true, @@ -571,6 +570,184 @@ describe.each([ expect(sum.calculationType).toEqual(CalculationType.SUM) expect(sum.field).toEqual("Price") }) + + it("cannot create a view with calculation fields unless it has the right type", async () => { + await config.api.viewV2.create( + { + tableId: table._id!, + name: generator.guid(), + schema: { + sum: { + visible: true, + calculationType: CalculationType.SUM, + field: "Price", + }, + }, + }, + { + status: 400, + body: { + message: + "Calculation fields are not allowed in non-calculation views", + }, + } + ) + }) + + it("cannot create a calculation view with more than 5 aggregations", async () => { + await config.api.viewV2.create( + { + tableId: table._id!, + name: generator.guid(), + type: ViewV2Type.CALCULATION, + schema: { + sum: { + visible: true, + calculationType: CalculationType.SUM, + field: "Price", + }, + count: { + visible: true, + calculationType: CalculationType.COUNT, + field: "Price", + }, + min: { + visible: true, + calculationType: CalculationType.MIN, + field: "Price", + }, + max: { + visible: true, + calculationType: CalculationType.MAX, + field: "Price", + }, + avg: { + visible: true, + calculationType: CalculationType.AVG, + field: "Price", + }, + sum2: { + visible: true, + calculationType: CalculationType.SUM, + field: "Price", + }, + }, + }, + { + status: 400, + body: { + message: "Calculation views can only have a maximum of 5 fields", + }, + } + ) + }) + + it("cannot create a calculation view with duplicate calculations", async () => { + await config.api.viewV2.create( + { + tableId: table._id!, + name: generator.guid(), + type: ViewV2Type.CALCULATION, + schema: { + sum: { + visible: true, + calculationType: CalculationType.SUM, + field: "Price", + }, + sum2: { + visible: true, + calculationType: CalculationType.SUM, + field: "Price", + }, + }, + }, + { + status: 400, + body: { + message: + 'Duplicate calculation on field "Price", calculation type "sum"', + }, + } + ) + }) + + it("finds duplicate counts", async () => { + await config.api.viewV2.create( + { + tableId: table._id!, + name: generator.guid(), + type: ViewV2Type.CALCULATION, + schema: { + count: { + visible: true, + calculationType: CalculationType.COUNT, + }, + count2: { + visible: true, + calculationType: CalculationType.COUNT, + }, + }, + }, + { + status: 400, + body: { + message: + 'Duplicate calculation on field "*", calculation type "count"', + }, + } + ) + }) + + it("finds duplicate count distincts", async () => { + await config.api.viewV2.create( + { + tableId: table._id!, + name: generator.guid(), + type: ViewV2Type.CALCULATION, + schema: { + count: { + visible: true, + calculationType: CalculationType.COUNT, + distinct: true, + field: "Price", + }, + count2: { + visible: true, + calculationType: CalculationType.COUNT, + distinct: true, + field: "Price", + }, + }, + }, + { + status: 400, + body: { + message: + 'Duplicate calculation on field "Price", calculation type "count"', + }, + } + ) + }) + + it("does not confuse counts and count distincts in the duplicate check", async () => { + await config.api.viewV2.create({ + tableId: table._id!, + name: generator.guid(), + type: ViewV2Type.CALCULATION, + schema: { + count: { + visible: true, + calculationType: CalculationType.COUNT, + }, + count2: { + visible: true, + calculationType: CalculationType.COUNT, + distinct: true, + field: "Price", + }, + }, + }) + }) }) describe("update", () => { @@ -615,7 +792,9 @@ describe.each([ it("can update all fields", async () => { const tableId = table._id! - const updatedData: Required> = { + const updatedData: Required< + Omit + > = { version: view.version, id: view.id, tableId, @@ -643,9 +822,6 @@ describe.each([ readonly: true, }, }, - uiMetadata: { - foo: "bar", - }, } await config.api.viewV2.update(updatedData) @@ -830,6 +1006,32 @@ describe.each([ ) }) + it("cannot update view type after creation", async () => { + const view = await config.api.viewV2.create({ + tableId: table._id!, + name: generator.guid(), + schema: { + id: { visible: true }, + Price: { + visible: true, + }, + }, + }) + + await config.api.viewV2.update( + { + ...view, + type: ViewV2Type.CALCULATION, + }, + { + status: 400, + body: { + message: "Cannot update view type after creation", + }, + } + ) + }) + isInternal && it("updating schema will only validate modified field", async () => { let view = await config.api.viewV2.create({ @@ -902,6 +1104,7 @@ describe.each([ view = await config.api.viewV2.create({ tableId: table._id!, name: generator.guid(), + type: ViewV2Type.CALCULATION, schema: { country: { visible: true, @@ -1024,6 +1227,26 @@ describe.each([ ) }) + it("can add a new group by field that is invisible, even if required on the table", async () => { + view.schema!.name = { visible: false } + await config.api.viewV2.update(view) + + const { rows } = await config.api.row.search(view.id) + expect(rows).toHaveLength(2) + expect(rows).toEqual( + expect.arrayContaining([ + { + country: "USA", + age: 65, + }, + { + country: "UK", + age: 61, + }, + ]) + ) + }) + it("can add a new calculation field", async () => { view.schema!.count = { visible: true, @@ -2639,6 +2862,7 @@ describe.each([ it("should be able to search by calculations", async () => { const view = await config.api.viewV2.create({ tableId: table._id!, + type: ViewV2Type.CALCULATION, name: generator.guid(), schema: { "Quantity Sum": { @@ -2673,6 +2897,7 @@ describe.each([ const view = await config.api.viewV2.create({ tableId: table._id!, name: generator.guid(), + type: ViewV2Type.CALCULATION, schema: { quantity: { visible: true, @@ -2711,6 +2936,7 @@ describe.each([ const view = await config.api.viewV2.create({ tableId: table._id!, name: generator.guid(), + type: ViewV2Type.CALCULATION, schema: { aggregate: { visible: true, @@ -2771,6 +2997,7 @@ describe.each([ const view = await config.api.viewV2.create({ tableId: table._id!, name: generator.guid(), + type: ViewV2Type.CALCULATION, schema: { count: { visible: true, @@ -2805,6 +3032,7 @@ describe.each([ { tableId: table._id!, name: generator.guid(), + type: ViewV2Type.CALCULATION, schema: { count: { visible: true, @@ -2853,6 +3081,7 @@ describe.each([ const view = await config.api.viewV2.create({ tableId: table._id!, name: generator.guid(), + type: ViewV2Type.CALCULATION, schema: { sum: { visible: true, diff --git a/packages/server/src/environment.ts b/packages/server/src/environment.ts index 585eb6a7f2..45d675ec3f 100644 --- a/packages/server/src/environment.ts +++ b/packages/server/src/environment.ts @@ -83,6 +83,7 @@ const environment = { PLUGINS_DIR: process.env.PLUGINS_DIR || DEFAULTS.PLUGINS_DIR, MAX_IMPORT_SIZE_MB: process.env.MAX_IMPORT_SIZE_MB, SESSION_EXPIRY_SECONDS: process.env.SESSION_EXPIRY_SECONDS, + XSS_SAFE_MODE: process.env.XSS_SAFE_MODE, // SQL SQL_MAX_ROWS: process.env.SQL_MAX_ROWS, SQL_LOGGING_ENABLE: process.env.SQL_LOGGING_ENABLE, diff --git a/packages/server/src/jsRunner/tests/jsRunner.spec.ts b/packages/server/src/jsRunner/tests/jsRunner.spec.ts index c7ab86f014..006df19fa6 100644 --- a/packages/server/src/jsRunner/tests/jsRunner.spec.ts +++ b/packages/server/src/jsRunner/tests/jsRunner.spec.ts @@ -8,7 +8,6 @@ import { init } from ".." import TestConfiguration from "../../tests/utilities/TestConfiguration" const DATE = "2021-01-21T12:00:00" - tk.freeze(DATE) describe("jsRunner (using isolated-vm)", () => { @@ -48,6 +47,33 @@ describe("jsRunner (using isolated-vm)", () => { ).toEqual("ReferenceError: process is not defined") }) + it("should not allow the context to be mutated", async () => { + const context = { array: [1] } + const result = await processJS( + ` + const array = $("array"); + array.push(2); + return array[1] + `, + context + ) + expect(result).toEqual(2) + expect(context.array).toEqual([1]) + }) + + it("should copy values whenever returning them from $", async () => { + const context = { array: [1] } + const result = await processJS( + ` + $("array").push(2); + return $("array")[1]; + `, + context + ) + expect(result).toEqual(undefined) + expect(context.array).toEqual([1]) + }) + describe("helpers", () => { runJsHelpersTests({ funcWrap: (func: any) => config.doInContext(config.getAppId(), func), diff --git a/packages/server/src/sdk/app/rows/tests/utils.spec.ts b/packages/server/src/sdk/app/rows/tests/utils.spec.ts index 548b2b6bc9..516334d31d 100644 --- a/packages/server/src/sdk/app/rows/tests/utils.spec.ts +++ b/packages/server/src/sdk/app/rows/tests/utils.spec.ts @@ -8,6 +8,7 @@ import { import { generateTableID } from "../../../../db/utils" import { validate } from "../utils" import { generator } from "@budibase/backend-core/tests" +import { withEnv } from "../../../../environment" describe("validate", () => { const hour = () => generator.hour().toString().padStart(2, "0") @@ -332,4 +333,46 @@ describe("validate", () => { }) }) }) + + describe("XSS Safe mode", () => { + const getTable = (): Table => ({ + type: "table", + _id: generateTableID(), + name: "table", + sourceId: INTERNAL_TABLE_SOURCE_ID, + sourceType: TableSourceType.INTERNAL, + schema: { + text: { + name: "sometext", + type: FieldType.STRING, + }, + }, + }) + it.each([ + "SELECT * FROM users WHERE username = 'admin' --", + "SELECT * FROM users WHERE id = 1; DROP TABLE users;", + "1' OR '1' = '1", + "' OR 'a' = 'a", + "", + '">', + "", + "
Hover over me!
", + "'; EXEC sp_msforeachtable 'DROP TABLE ?'; --", + "{alert('Injected')}", + "UNION SELECT * FROM users", + "INSERT INTO users (username, password) VALUES ('admin', 'password')", + "/* This is a comment */ SELECT * FROM users", + '', + ])("test potentially unsafe input: %s", async input => { + await withEnv({ XSS_SAFE_MODE: "1" }, async () => { + const table = getTable() + const row = { text: input } + const output = await validate({ source: table, row }) + expect(output.valid).toBe(false) + expect(output.errors).toStrictEqual({ + text: ["Input not sanitised - potentially vulnerable to XSS"], + }) + }) + }) + }) }) diff --git a/packages/server/src/sdk/app/rows/utils.ts b/packages/server/src/sdk/app/rows/utils.ts index 6ef4dcbc8e..bded6a7a18 100644 --- a/packages/server/src/sdk/app/rows/utils.ts +++ b/packages/server/src/sdk/app/rows/utils.ts @@ -22,6 +22,7 @@ import { extractViewInfoFromID, isRelationshipColumn } from "../../../db/utils" import { isSQL } from "../../../integrations/utils" import { docIds, sql } from "@budibase/backend-core" import { getTableFromSource } from "../../../api/controllers/row/utils" +import env from "../../../environment" const SQL_CLIENT_SOURCE_MAP: Record = { [SourceName.POSTGRES]: SqlClient.POSTGRES, @@ -43,6 +44,9 @@ const SQL_CLIENT_SOURCE_MAP: Record = { [SourceName.BUDIBASE]: undefined, } +const XSS_INPUT_REGEX = + /[<>;"'(){}]|--|\/\*|\*\/|union|select|insert|drop|delete|update|exec|script/i + export function getSQLClient(datasource: Datasource): SqlClient { if (!isSQL(datasource)) { throw new Error("Cannot get SQL Client for non-SQL datasource") @@ -222,6 +226,15 @@ export async function validate({ } else { res = validateJs.single(row[fieldName], constraints) } + + if (env.XSS_SAFE_MODE && typeof row[fieldName] === "string") { + if (XSS_INPUT_REGEX.test(row[fieldName])) { + errors[fieldName] = [ + "Input not sanitised - potentially vulnerable to XSS", + ] + } + } + if (res) errors[fieldName] = res } return { valid: Object.keys(errors).length === 0, errors } diff --git a/packages/server/src/sdk/app/views/external.ts b/packages/server/src/sdk/app/views/external.ts index 3afd7e9bf9..b69ac0b9eb 100644 --- a/packages/server/src/sdk/app/views/external.ts +++ b/packages/server/src/sdk/app/views/external.ts @@ -70,6 +70,9 @@ export async function update(tableId: string, view: ViewV2): Promise { if (!existingView || !existingView.name) { throw new HTTPError(`View ${view.id} not found in table ${tableId}`, 404) } + if (isV2(existingView) && existingView.type !== view.type) { + throw new HTTPError(`Cannot update view type after creation`, 400) + } delete views[existingView.name] views[view.name] = view diff --git a/packages/server/src/sdk/app/views/index.ts b/packages/server/src/sdk/app/views/index.ts index f8b6f83e90..853b4eb37c 100644 --- a/packages/server/src/sdk/app/views/index.ts +++ b/packages/server/src/sdk/app/views/index.ts @@ -62,11 +62,31 @@ async function guardCalculationViewSchema( view: Omit ) { const calculationFields = helpers.views.calculationFields(view) - for (const calculationFieldName of Object.keys(calculationFields)) { - const schema = calculationFields[calculationFieldName] + + if (Object.keys(calculationFields).length > 5) { + throw new HTTPError( + "Calculation views can only have a maximum of 5 fields", + 400 + ) + } + + const seen: Record> = {} + + for (const name of Object.keys(calculationFields)) { + const schema = calculationFields[name] const isCount = schema.calculationType === CalculationType.COUNT const isDistinct = isCount && "distinct" in schema && schema.distinct + const field = isCount && !isDistinct ? "*" : schema.field + if (seen[field]?.[schema.calculationType]) { + throw new HTTPError( + `Duplicate calculation on field "${field}", calculation type "${schema.calculationType}"`, + 400 + ) + } + seen[field] ??= {} as Record + seen[field][schema.calculationType] = true + // Count fields that aren't distinct don't need to reference another field, // so we don't validate it. if (isCount && !isDistinct) { @@ -76,14 +96,14 @@ async function guardCalculationViewSchema( const targetSchema = table.schema[schema.field] if (!targetSchema) { throw new HTTPError( - `Calculation field "${calculationFieldName}" references field "${schema.field}" which does not exist in the table schema`, + `Calculation field "${name}" references field "${schema.field}" which does not exist in the table schema`, 400 ) } if (!isCount && !helpers.schema.isNumeric(targetSchema)) { throw new HTTPError( - `Calculation field "${calculationFieldName}" references field "${schema.field}" which is not a numeric field`, + `Calculation field "${name}" references field "${schema.field}" which is not a numeric field`, 400 ) } @@ -109,10 +129,21 @@ async function guardViewSchema( if (helpers.views.isCalculationView(view)) { await guardCalculationViewSchema(table, view) + } else { + if (helpers.views.hasCalculationFields(view)) { + throw new HTTPError( + "Calculation fields are not allowed in non-calculation views", + 400 + ) + } } await checkReadonlyFields(table, view) - checkRequiredFields(table, view) + + if (!helpers.views.isCalculationView(view)) { + checkRequiredFields(table, view) + } + checkDisplayField(view) } diff --git a/packages/server/src/sdk/app/views/internal.ts b/packages/server/src/sdk/app/views/internal.ts index 19a9f6ab14..96b41bffe8 100644 --- a/packages/server/src/sdk/app/views/internal.ts +++ b/packages/server/src/sdk/app/views/internal.ts @@ -59,6 +59,10 @@ export async function update(tableId: string, view: ViewV2): Promise { throw new HTTPError(`View ${view.id} not found in table ${tableId}`, 404) } + if (isV2(existingView) && existingView.type !== view.type) { + throw new HTTPError(`Cannot update view type after creation`, 400) + } + delete table.views[existingView.name] table.views[view.name] = view await db.put(table) diff --git a/packages/shared-core/src/helpers/views.ts b/packages/shared-core/src/helpers/views.ts index f41c66adc8..0113375adf 100644 --- a/packages/shared-core/src/helpers/views.ts +++ b/packages/shared-core/src/helpers/views.ts @@ -3,6 +3,7 @@ import { ViewCalculationFieldMetadata, ViewFieldMetadata, ViewV2, + ViewV2Type, } from "@budibase/types" import { pickBy } from "lodash" @@ -21,6 +22,10 @@ export function isBasicViewField( type UnsavedViewV2 = Omit export function isCalculationView(view: UnsavedViewV2) { + return view.type === ViewV2Type.CALCULATION +} + +export function hasCalculationFields(view: UnsavedViewV2) { return Object.values(view.schema || {}).some(isCalculationField) } diff --git a/packages/string-templates/src/helpers/javascript.ts b/packages/string-templates/src/helpers/javascript.ts index 2f850b0533..4fb1329196 100644 --- a/packages/string-templates/src/helpers/javascript.ts +++ b/packages/string-templates/src/helpers/javascript.ts @@ -1,13 +1,14 @@ -import { atob, isJSAllowed } from "../utilities" -import cloneDeep from "lodash/fp/cloneDeep" +import { atob, isBackendService, isJSAllowed } from "../utilities" import { LITERAL_MARKER } from "../helpers/constants" import { getJsHelperList } from "./list" import { iifeWrapper } from "../iife" import { JsTimeoutError, UserScriptError } from "../errors" +import { cloneDeep } from "lodash/fp" // The method of executing JS scripts depends on the bundle being built. // This setter is used in the entrypoint (either index.js or index.mjs). -let runJS: ((js: string, context: any) => any) | undefined = undefined +let runJS: ((js: string, context: Record) => any) | undefined = + undefined export const setJSRunner = (runner: typeof runJS) => (runJS = runner) export const removeJSRunner = () => { @@ -31,9 +32,19 @@ const removeSquareBrackets = (value: string) => { return value } +const isReservedKey = (key: string) => + key === "snippets" || + key === "helpers" || + key.startsWith("snippets.") || + key.startsWith("helpers.") + // Our context getter function provided to JS code as $. // Extracts a value from context. const getContextValue = (path: string, context: any) => { + // We populate `snippets` ourselves, don't allow access to it. + if (isReservedKey(path)) { + return undefined + } const literalStringRegex = /^(["'`]).*\1$/ let data = context // check if it's a literal string - just return path if its quoted @@ -46,6 +57,7 @@ const getContextValue = (path: string, context: any) => { } data = data[removeSquareBrackets(key)] }) + return data } @@ -67,10 +79,23 @@ export function processJS(handlebars: string, context: any) { snippetMap[snippet.name] = snippet.code } - // Our $ context function gets a value from context. - // We clone the context to avoid mutation in the binding affecting real - // app context. - const clonedContext = cloneDeep({ ...context, snippets: null }) + let clonedContext: Record + if (isBackendService()) { + // On the backned, values are copied across the isolated-vm boundary and + // so we don't need to do any cloning here. This does create a fundamental + // difference in how JS executes on the frontend vs the backend, e.g. + // consider this snippet: + // + // $("array").push(2) + // return $("array")[1] + // + // With the context of `{ array: [1] }`, the backend will return + // `undefined` whereas the frontend will return `2`. We should fix this. + clonedContext = context + } else { + clonedContext = cloneDeep(context) + } + const sandboxContext = { $: (path: string) => getContextValue(path, clonedContext), helpers: getJsHelperList(), diff --git a/packages/string-templates/src/index.ts b/packages/string-templates/src/index.ts index c2be63978d..7246bc4942 100644 --- a/packages/string-templates/src/index.ts +++ b/packages/string-templates/src/index.ts @@ -1,4 +1,4 @@ -import { Context, createContext, runInNewContext } from "vm" +import { createContext, runInNewContext } from "vm" import { create, TemplateDelegate } from "handlebars" import { registerAll, registerMinimum } from "./helpers/index" import { postprocess, preprocess } from "./processors" @@ -455,21 +455,14 @@ export function convertToJS(hbs: string) { export { JsTimeoutError, UserScriptError } from "./errors" -export function defaultJSSetup() { - if (!isBackendService()) { - /** - * Use polyfilled vm to run JS scripts in a browser Env - */ - setJSRunner((js: string, context: Context) => { - context = { - ...context, - alert: undefined, - setInterval: undefined, - setTimeout: undefined, - } - createContext(context) +export function browserJSSetup() { + /** + * Use polyfilled vm to run JS scripts in a browser Env + */ + setJSRunner((js: string, context: Record) => { + createContext(context) - const wrappedJs = ` + const wrappedJs = ` result = { result: null, error: null, @@ -484,12 +477,17 @@ export function defaultJSSetup() { result; ` - const result = runInNewContext(wrappedJs, context, { timeout: 1000 }) - if (result.error) { - throw new UserScriptError(result.error) - } - return result.result - }) + const result = runInNewContext(wrappedJs, context, { timeout: 1000 }) + if (result.error) { + throw new UserScriptError(result.error) + } + return result.result + }) +} + +export function defaultJSSetup() { + if (!isBackendService()) { + browserJSSetup() } else { removeJSRunner() } diff --git a/packages/string-templates/src/utilities.ts b/packages/string-templates/src/utilities.ts index 18e0926b96..779bef3735 100644 --- a/packages/string-templates/src/utilities.ts +++ b/packages/string-templates/src/utilities.ts @@ -4,7 +4,14 @@ export const FIND_HBS_REGEX = /{{([^{].*?)}}/g export const FIND_ANY_HBS_REGEX = /{?{{([^{].*?)}}}?/g export const FIND_TRIPLE_HBS_REGEX = /{{{([^{].*?)}}}/g +const isJest = () => typeof jest !== "undefined" + export const isBackendService = () => { + // We consider the tests for string-templates to be frontend, so that they + // test the frontend JS functionality. + if (isJest()) { + return false + } return typeof window === "undefined" } diff --git a/packages/string-templates/test/javascript.spec.ts b/packages/string-templates/test/javascript.spec.ts index 99e6ee122a..9134005acb 100644 --- a/packages/string-templates/test/javascript.spec.ts +++ b/packages/string-templates/test/javascript.spec.ts @@ -1,7 +1,13 @@ -import vm from "vm" - -import { processStringSync, encodeJSBinding, setJSRunner } from "../src/index" +import { + processStringSync, + encodeJSBinding, + defaultJSSetup, +} from "../src/index" import { UUID_REGEX } from "./constants" +import tk from "timekeeper" + +const DATE = "2021-01-21T12:00:00" +tk.freeze(DATE) const processJS = (js: string, context?: object): any => { return processStringSync(encodeJSBinding(js), context) @@ -9,9 +15,7 @@ const processJS = (js: string, context?: object): any => { describe("Javascript", () => { beforeAll(() => { - setJSRunner((js, context) => { - return vm.runInNewContext(js, context, { timeout: 1000 }) - }) + defaultJSSetup() }) describe("Test the JavaScript helper", () => { @@ -118,8 +122,7 @@ describe("Javascript", () => { }) it("should handle errors", () => { - const output = processJS(`throw "Error"`) - expect(output).toBe("Error while executing JS") + expect(processJS(`throw "Error"`)).toEqual("Error") }) it("should timeout after one second", () => { @@ -127,16 +130,18 @@ describe("Javascript", () => { expect(output).toBe("Timed out while executing JS") }) - it("should prevent access to the process global", () => { - const output = processJS(`return process`) - expect(output).toBe("Error while executing JS") + it("should prevent access to the process global", async () => { + expect(processJS(`return process`)).toEqual( + "ReferenceError: process is not defined" + ) }) }) describe("check JS helpers", () => { it("should error if using the format helper. not helpers.", () => { - const output = processJS(`return helper.toInt(4.3)`) - expect(output).toBe("Error while executing JS") + expect(processJS(`return helper.toInt(4.3)`)).toEqual( + "ReferenceError: helper is not defined" + ) }) it("should be able to use toInt", () => { @@ -156,4 +161,323 @@ describe("Javascript", () => { expect(output).toBe("Custom") }) }) + + describe("mutability", () => { + it("should not allow the context to be mutated", async () => { + const context = { array: [1] } + const result = await processJS( + ` + const array = $("array"); + array.push(2); + return array[1] + `, + context + ) + expect(result).toEqual(2) + expect(context.array).toEqual([1]) + }) + }) + + describe("malice", () => { + it("should not be able to call JS functions", () => { + expect(processJS(`return alert("hello")`)).toEqual( + "ReferenceError: alert is not defined" + ) + + expect(processJS(`return prompt("hello")`)).toEqual( + "ReferenceError: prompt is not defined" + ) + + expect(processJS(`return confirm("hello")`)).toEqual( + "ReferenceError: confirm is not defined" + ) + + expect(processJS(`return setTimeout(() => {}, 1000)`)).toEqual( + "ReferenceError: setTimeout is not defined" + ) + + expect(processJS(`return setInterval(() => {}, 1000)`)).toEqual( + "ReferenceError: setInterval is not defined" + ) + }) + }) + + // the test cases here were extracted from templates/real world examples of JS in Budibase + describe("real test cases from Budicloud", () => { + const context = { + "Unit Value": 2, + Quantity: 1, + } + it("handle test case 1", async () => { + const result = await processJS( + ` + var Gross = $("[Unit Value]") * $("[Quantity]") + return Gross.toFixed(2)`, + context + ) + expect(result).toBeDefined() + expect(result).toBe("2.00") + }) + + it("handle test case 2", async () => { + const todayDate = new Date() + // add a year and a month + todayDate.setMonth(new Date().getMonth() + 1) + todayDate.setFullYear(todayDate.getFullYear() + 1) + const context = { + "Purchase Date": DATE, + today: todayDate.toISOString(), + } + const result = await processJS( + ` + var purchase = new Date($("[Purchase Date]")); + let purchaseyear = purchase.getFullYear(); + let purchasemonth = purchase.getMonth(); + + var today = new Date($("today")); + let todayyear = today.getFullYear(); + let todaymonth = today.getMonth(); + + var age = todayyear - purchaseyear + + if (((todaymonth - purchasemonth) < 6) == true){ + return age + } + `, + context + ) + expect(result).toBeDefined() + expect(result).toBe(1) + }) + + it("should handle test case 3", async () => { + const context = { + Escalate: true, + "Budget ($)": 1100, + } + const result = await processJS( + ` + if ($("[Escalate]") == true) { + if ($("Budget ($)") <= 1000) + {return 2;} + if ($("Budget ($)") > 1000) + {return 3;} + } + else { + if ($("Budget ($)") <= 1000) + {return 1;} + if ($("Budget ($)") > 1000) + if ($("Budget ($)") < 10000) + {return 2;} + else + {return 3} + } + `, + context + ) + expect(result).toBeDefined() + expect(result).toBe(3) + }) + + it("should handle test case 4", async () => { + const context = { + "Time Sheets": ["a", "b"], + } + const result = await processJS( + ` + let hours = 0 + if (($("[Time Sheets]") != null) == true){ + for (i = 0; i < $("[Time Sheets]").length; i++){ + let hoursLogged = "Time Sheets." + i + ".Hours" + hours += $(hoursLogged) + } + return hours + } + if (($("[Time Sheets]") != null) == false){ + return hours + } + `, + context + ) + expect(result).toBeDefined() + expect(result).toBe("0ab") + }) + + it("should handle test case 5", async () => { + const context = { + change: JSON.stringify({ a: 1, primaryDisplay: "a" }), + previous: JSON.stringify({ a: 2, primaryDisplay: "b" }), + } + const result = await processJS( + ` + let change = $("[change]") ? JSON.parse($("[change]")) : {} + let previous = $("[previous]") ? JSON.parse($("[previous]")) : {} + + function simplifyLink(originalKey, value, parent) { + if (Array.isArray(value)) { + if (value.filter(item => Object.keys(item || {}).includes("primaryDisplay")).length > 0) { + parent[originalKey] = value.map(link => link.primaryDisplay) + } + } + } + + for (let entry of Object.entries(change)) { + simplifyLink(entry[0], entry[1], change) + } + for (let entry of Object.entries(previous)) { + simplifyLink(entry[0], entry[1], previous) + } + + let diff = Object.fromEntries(Object.entries(change).filter(([k, v]) => previous[k]?.toString() !== v?.toString())) + + delete diff.audit_change + delete diff.audit_previous + delete diff._id + delete diff._rev + delete diff.tableId + delete diff.audit + + for (let entry of Object.entries(diff)) { + simplifyLink(entry[0], entry[1], diff) + } + + return JSON.stringify(change)?.replaceAll(",\\"", ",\\n\\t\\"").replaceAll("{\\"", "{\\n\\t\\"").replaceAll("}", "\\n}") + `, + context + ) + expect(result).toBe(`{\n\t"a":1,\n\t"primaryDisplay":"a"\n}`) + }) + + it("should handle test case 6", async () => { + const context = { + "Join Date": DATE, + } + const result = await processJS( + ` + var rate = 5; + var today = new Date(); + + // comment + function monthDiff(dateFrom, dateTo) { + return dateTo.getMonth() - dateFrom.getMonth() + + (12 * (dateTo.getFullYear() - dateFrom.getFullYear())) + } + var serviceMonths = monthDiff( new Date($("[Join Date]")), today); + var serviceYears = serviceMonths / 12; + + if (serviceYears >= 1 && serviceYears < 5){ + rate = 10; + } + if (serviceYears >= 5 && serviceYears < 10){ + rate = 15; + } + if (serviceYears >= 10){ + rate = 15; + rate += 0.5 * (Number(serviceYears.toFixed(0)) - 10); + } + return rate; + `, + context + ) + expect(result).toBe(10) + }) + + it("should handle test case 7", async () => { + const context = { + "P I": "Pass", + "PA I": "Pass", + "F I": "Fail", + "V I": "Pass", + } + const result = await processJS( + `if (($("[P I]") == "Pass") == true) + if (($("[ P I]") == "Pass") == true) + if (($("[F I]") == "Pass") == true) + if (($("[V I]") == "Pass") == true) + {return "Pass"} + + if (($("[PA I]") == "Fail") == true) + {return "Fail"} + if (($("[ P I]") == "Fail") == true) + {return "Fail"} + if (($("[F I]") == "Fail") == true) + {return "Fail"} + if (($("[V I]") == "Fail") == true) + {return "Fail"} + + else + {return ""}`, + context + ) + expect(result).toBe("Fail") + }) + + it("should handle test case 8", async () => { + const context = { + "T L": [{ Hours: 10 }], + "B H": 50, + } + const result = await processJS( + `var totalHours = 0; + if (($("[T L]") != null) == true){ + for (let i = 0; i < ($("[T L]").length); i++){ + var individualHours = "T L." + i + ".Hours"; + var hoursNum = Number($(individualHours)); + totalHours += hoursNum; + } + return totalHours.toFixed(2); + } + if (($("[T L]") != null) == false) { + return totalHours.toFixed(2); + } + `, + context + ) + expect(result).toBe("10.00") + }) + + it("should handle test case 9", async () => { + const context = { + "T L": [{ Hours: 10 }], + "B H": 50, + } + const result = await processJS( + `var totalHours = 0; + if (($("[T L]") != null) == true){ + for (let i = 0; i < ($("[T L]").length); i++){ + var individualHours = "T L." + i + ".Hours"; + var hoursNum = Number($(individualHours)); + totalHours += hoursNum; + } + return ($("[B H]") - totalHours).toFixed(2); + } + if (($("[T L]") != null) == false) { + return ($("[B H]") - totalHours).toFixed(2); + }`, + context + ) + expect(result).toBe("40.00") + }) + + it("should handle test case 10", async () => { + const context = { + "F F": [{ "F S": 10 }], + } + const result = await processJS( + `var rating = 0; + + if ($("[F F]") != null){ + for (i = 0; i < $("[F F]").length; i++){ + var individualRating = $("F F." + i + ".F S"); + rating += individualRating; + } + rating = (rating / $("[F F]").length); + } + return rating; + `, + context + ) + expect(result).toBe(10) + }) + }) }) diff --git a/packages/string-templates/test/manifest.spec.ts b/packages/string-templates/test/manifest.spec.ts index de0e3355fd..2d09bc7871 100644 --- a/packages/string-templates/test/manifest.spec.ts +++ b/packages/string-templates/test/manifest.spec.ts @@ -1,5 +1,3 @@ -import vm from "vm" - jest.mock("@budibase/handlebars-helpers/lib/math", () => { const actual = jest.requireActual("@budibase/handlebars-helpers/lib/math") @@ -17,7 +15,7 @@ jest.mock("@budibase/handlebars-helpers/lib/uuid", () => { } }) -import { processString, setJSRunner } from "../src/index" +import { defaultJSSetup, processString } from "../src/index" import tk from "timekeeper" import { getParsedManifest, runJsHelpersTests } from "./utils" @@ -32,9 +30,7 @@ describe("manifest", () => { const manifest = getParsedManifest() beforeAll(() => { - setJSRunner((js, context) => { - return vm.runInNewContext(js, context, { timeout: 1000 }) - }) + defaultJSSetup() }) describe("examples are valid", () => { diff --git a/packages/types/src/api/account/accounts.ts b/packages/types/src/api/account/accounts.ts index 1be506e14e..e05c1d0bf3 100644 --- a/packages/types/src/api/account/accounts.ts +++ b/packages/types/src/api/account/accounts.ts @@ -12,7 +12,6 @@ export interface CreateAccountRequest { name?: string password: string provider?: AccountSSOProvider - thirdPartyProfile: object } export interface SearchAccountsRequest { diff --git a/packages/types/src/documents/account/account.ts b/packages/types/src/documents/account/account.ts index 61792a7f47..aac5bf2d20 100644 --- a/packages/types/src/documents/account/account.ts +++ b/packages/types/src/documents/account/account.ts @@ -7,10 +7,9 @@ export interface CreateAccount { tenantId: string hosting: Hosting authType: AuthType + accountName: string // optional fields - for sso based sign ups registrationStep?: string - // profile - tenantName?: string name?: string size?: string profession?: string @@ -20,11 +19,6 @@ export interface CreatePassswordAccount extends CreateAccount { password: string } -export interface CreateVerifiableSSOAccount extends CreateAccount { - provider?: AccountSSOProvider - thirdPartyProfile?: any -} - export const isCreatePasswordAccount = ( account: CreateAccount ): account is CreatePassswordAccount => account.authType === AuthType.PASSWORD @@ -56,6 +50,7 @@ export interface Account extends CreateAccount { providerType?: AccountSSOProviderType quotaUsage?: QuotaUsage offlineLicenseToken?: string + tenantName?: string } export interface PasswordAccount extends Account { @@ -103,8 +98,6 @@ export interface AccountSSO { provider: AccountSSOProvider providerType: AccountSSOProviderType oauth2?: OAuthTokens - pictureUrl?: string - thirdPartyProfile: any // TODO: define what the google profile looks like } export type SSOAccount = (Account | CloudAccount) & AccountSSO diff --git a/packages/types/src/documents/app/view.ts b/packages/types/src/documents/app/view.ts index c58852ecea..1b2372da85 100644 --- a/packages/types/src/documents/app/view.ts +++ b/packages/types/src/documents/app/view.ts @@ -79,10 +79,15 @@ export enum CalculationType { MAX = "max", } +export enum ViewV2Type { + CALCULATION = "calculation", +} + export interface ViewV2 { version: 2 id: string name: string + type?: ViewV2Type primaryDisplay?: string tableId: string query?: LegacyFilter[] | SearchFilters @@ -94,7 +99,6 @@ export interface ViewV2 { type?: SortType } schema?: ViewV2Schema - uiMetadata?: Record } export type ViewV2Schema = Record diff --git a/packages/types/src/documents/global/user.ts b/packages/types/src/documents/global/user.ts index 7605f013d1..a1c5b2506f 100644 --- a/packages/types/src/documents/global/user.ts +++ b/packages/types/src/documents/global/user.ts @@ -21,7 +21,6 @@ export interface UserSSO { provider: string // the individual provider e.g. Okta, Auth0, Google providerType: SSOProviderType oauth2?: OAuth2 - thirdPartyProfile?: SSOProfileJson profile?: { displayName?: string name?: { @@ -45,7 +44,6 @@ export interface User extends Document { userId?: string firstName?: string lastName?: string - pictureUrl?: string forceResetPassword?: boolean roles: UserRoles builder?: {