MirrorSave
/
budibase
mirror of https://github.com/Budibase/budibase.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Merge branch 'master' into feature/monolith-js-refactor

This commit is contained in:
deanhannigan 2024-02-12 10:15:11 +00:00 committed by GitHub
parent 43bc4f730d 7aa1176a36
commit 118f100cbb
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
95 changed files with 1722 additions and 1383 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
.eslintignore
View File

@ -10,4 +10,5 @@ packages/builder/.routify
packages/sdk/sdk
packages/account-portal/packages/server/build
packages/account-portal/packages/ui/.routify
packages/account-portal/packages/ui/build
packages/account-portal/packages/ui/build
**/*.ivm.bundle.js

5
.eslintrc.json
View File

@ -43,7 +43,8 @@
"no-useless-escape": "off",
"no-undef": "off",
"no-prototype-builtins": "off",
"local-rules/no-budibase-imports": "error"
"local-rules/no-budibase-imports": "error",
"local-rules/no-test-com": "error"
}
},
{
@ -53,7 +54,7 @@
"packages/frontend-core/**/*"
],
"rules": {
"no-console": ["error", { "allow": ["warn", "error", "debug"] } ]
"no-console": ["error", { "allow": ["warn", "error", "debug"] }]
}
}
],

3
.prettierignore
View File

@ -11,4 +11,5 @@ packages/sdk/sdk
packages/pro/coverage
packages/account-portal/packages/ui/build
packages/account-portal/packages/ui/.routify
packages/account-portal/packages/server/build
packages/account-portal/packages/server/build
**/*.ivm.bundle.js

33
eslint-local-rules/index.js
View File

@ -18,4 +18,37 @@ module.exports = {
}
},
},
"no-test-com": {
meta: {
type: "problem",
docs: {
description:
"disallow the use of 'test.com' in strings and replace it with 'example.com'",
category: "Possible Errors",
recommended: false,
},
schema: [], // no options
fixable: "code", // Indicates that this rule supports automatic fixing
},
create: function (context) {
return {
Literal(node) {
if (
typeof node.value === "string" &&
node.value.includes("test.com")
) {
context.report({
node,
message:
"test.com is a privately owned domain and could point anywhere, use example.com instead.",
fix: function (fixer) {
const newText = node.raw.replace(/test\.com/g, "example.com")
return fixer.replaceText(node, newText)
},
})
}
},
}
},
},
}

2
hosting/single/Dockerfile
View File

@ -124,6 +124,8 @@ HEALTHCHECK --interval=15s --timeout=15s --start-period=45s CMD "/healthcheck.sh
# must set this just before running
ENV NODE_ENV=production
# this is required for isolated-vm to work on Node 20+
ENV NODE_OPTIONS="--no-node-snapshot"
WORKDIR /
CMD ["./runner.sh"]

2
lerna.json
View File

@ -1,5 +1,5 @@
{
"version": "2.17.7",
"version": "2.19.0",
"npmClient": "yarn",
"packages": [
"packages/*",

12
package.json
View File

@ -97,7 +97,17 @@
"@budibase/backend-core": "0.0.0",
"@budibase/shared-core": "0.0.0",
"@budibase/string-templates": "0.0.0",
"@budibase/types": "0.0.0"
"@budibase/types": "0.0.0",
"tough-cookie": "4.1.3",
"node-fetch": "2.6.7",
"semver": "7.5.3",
"http-cache-semantics": "4.1.1",
"msgpackr": "1.10.1",
"axios": "1.6.3",
"xml2js": "0.6.2",
"unset-value": "2.0.1",
"got": "13.0.0",
"passport": "0.6.0"
},
"engines": {
"node": ">=20.0.0 <21.0.0"

10
packages/backend-core/package.json
View File

@ -25,19 +25,19 @@
"@budibase/pouchdb-replication-stream": "1.2.10",
"@budibase/shared-core": "0.0.0",
"@budibase/types": "0.0.0",
"@techpass/passport-openidconnect": "0.3.2",
"@govtechsg/passport-openidconnect": "^1.0.2",
"aws-cloudfront-sign": "3.0.2",
"aws-sdk": "2.1030.0",
"bcrypt": "5.1.0",
"bcryptjs": "2.4.3",
"bull": "4.10.1",
"correlation-id": "4.0.0",
"dd-trace": "5.0.0",
"dd-trace": "5.2.0",
"dotenv": "16.0.1",
"ioredis": "5.3.2",
"joi": "17.6.0",
"jsonwebtoken": "9.0.2",
"koa-passport": "4.1.4",
"koa-passport": "^6.0.0",
"koa-pino-logger": "4.0.0",
"lodash": "4.17.21",
"node-fetch": "2.6.7",
@ -52,9 +52,9 @@
"redlock": "4.2.0",
"rotating-file-stream": "3.1.0",
"sanitize-s3-objectkey": "0.0.1",
"semver": "7.3.7",
"semver": "^7.5.4",
"tar-fs": "2.1.1",
"uuid": "8.3.2"
"uuid": "^8.3.2"
},
"devDependencies": {
"@shopify/jest-koa-mocks": "5.1.1",

3
packages/backend-core/src/context/types.ts
View File

@ -1,4 +1,4 @@
import { IdentityContext } from "@budibase/types"
import { IdentityContext, VM } from "@budibase/types"
import { ExecutionTimeTracker } from "../timers"
// keep this out of Budibase types, don't want to expose context info
@ -11,4 +11,5 @@ export type ContextMap = {
automationId?: string
isMigrating?: boolean
jsExecutionTracker?: ExecutionTimeTracker
vm?: VM
}

3
packages/backend-core/src/objectStore/objectStore.ts
View File

@ -255,7 +255,8 @@ export async function listAllObjects(bucketName: string, path: string) {
objects = objects.concat(response.Contents)
}
isTruncated = !!response.IsTruncated
} while (isTruncated)
token = response.NextContinuationToken
} while (isTruncated && token)
return objects
}

13
packages/backend-core/src/queue/queue.ts
View File

@ -2,7 +2,7 @@ import env from "../environment"
import { getRedisOptions } from "../redis/utils"
import { JobQueue } from "./constants"
import InMemoryQueue from "./inMemoryQueue"
import BullQueue, { QueueOptions } from "bull"
import BullQueue, { QueueOptions, JobOptions } from "bull"
import { addListeners, StalledFn } from "./listeners"
import { Duration } from "../utils"
import * as timers from "../timers"
@ -24,17 +24,24 @@ async function cleanup() {
export function createQueue<T>(
jobQueue: JobQueue,
opts: { removeStalledCb?: StalledFn } = {}
opts: {
removeStalledCb?: StalledFn
maxStalledCount?: number
jobOptions?: JobOptions
} = {}
): BullQueue.Queue<T> {
const redisOpts = getRedisOptions()
const queueConfig: QueueOptions = {
redis: redisOpts,
settings: {
maxStalledCount: 0,
maxStalledCount: opts.maxStalledCount ? opts.maxStalledCount : 0,
lockDuration: QUEUE_LOCK_MS,
lockRenewTime: QUEUE_LOCK_RENEW_INTERNAL_MS,
},
}
if (opts.jobOptions) {
queueConfig.defaultJobOptions = opts.jobOptions
}
let queue: any
if (!env.isTest()) {
queue = new BullQueue(jobQueue, queueConfig)

8
packages/backend-core/src/utils/tests/utils.spec.ts
View File

@ -44,11 +44,11 @@ describe("utils", () => {
it("gets appId from url", async () => {
await config.doInTenant(async () => {
const url = "http://test.com"
const url = "http://example.com"
env._set("PLATFORM_URL", url)
const ctx = structures.koa.newContext()
ctx.host = `${config.tenantId}.test.com`
ctx.host = `${config.tenantId}.example.com`
const expected = db.generateAppID(config.tenantId)
const app = structures.apps.app(expected)
@ -89,7 +89,7 @@ describe("utils", () => {
const ctx = structures.koa.newContext()
const expected = db.generateAppID()
ctx.request.headers = {
referer: `http://test.com/builder/app/${expected}/design/screen_123/screens`,
referer: `http://example.com/builder/app/${expected}/design/screen_123/screens`,
}
const actual = await utils.getAppIdFromCtx(ctx)
@ -100,7 +100,7 @@ describe("utils", () => {
const ctx = structures.koa.newContext()
const appId = db.generateAppID()
ctx.request.headers = {
referer: `http://test.com/foo/app/${appId}/bar`,
referer: `http://example.com/foo/app/${appId}/bar`,
}
const actual = await utils.getAppIdFromCtx(ctx)

2
packages/backend-core/tests/core/utilities/structures/common.ts
View File

@ -3,5 +3,5 @@ import { v4 as uuid } from "uuid"
export { v4 as uuid } from "uuid"
export const email = () => {
return `${uuid()}@test.com`
return `${uuid()}@example.com`
}

2
packages/backend-core/tests/core/utilities/structures/sso.ts
View File

@ -61,7 +61,7 @@ export function ssoProfile(user?: User): SSOProfile {
},
_json: {
email: user.email,
picture: "http://test.com",
picture: "http://example.com",
},
provider: generator.string(),
}

2
packages/backend-core/tests/core/utilities/structures/users.ts
View File

@ -25,7 +25,7 @@ export const user = (userProps?: Partial<Omit<User, "userId">>): User => {
roles: { app_test: "admin" },
firstName: generator.first(),
lastName: generator.last(),
pictureUrl: "http://test.com",
pictureUrl: "http://example.com",
tenantId: tenant.id(),
...userProps,
}

4
packages/bbui/src/Form/Core/EnvDropdown.svelte
View File

@ -184,7 +184,7 @@
{#if environmentVariablesEnabled}
<div on:click={() => showModal()} class="add-variable">
<svg
class="spectrum-Icon spectrum-Icon--sizeS "
class="spectrum-Icon spectrum-Icon--sizeS"
focusable="false"
aria-hidden="true"
>
@ -195,7 +195,7 @@
{:else}
<div on:click={() => handleUpgradePanel()} class="add-variable">
<svg
class="spectrum-Icon spectrum-Icon--sizeS "
class="spectrum-Icon spectrum-Icon--sizeS"
focusable="false"
aria-hidden="true"
>

13
packages/builder/src/components/integration/QueryViewer.svelte
View File

@ -41,6 +41,7 @@
let autoSchema = {}
let rows = []
let keys = {}
const parseQuery = query => {
modified = false
@ -143,8 +144,20 @@
const handleScroll = e => {
scrolling = e.target.scrollTop !== 0
}
async function handleKeyDown(evt) {
keys[evt.key] = true
if ((keys["Meta"] || keys["Control"]) && keys["Enter"]) {
await runQuery({ suppressErrors: false })
}
}
function handleKeyUp(evt) {
delete keys[evt.key]
}
</script>
<svelte:window on:keydown={handleKeyDown} on:keyup={handleKeyUp} />
<QueryViewerSavePromptModal
checkIsModified={() => checkIsModified(newQuery)}
attemptSave={() => runQuery({ suppressErrors: false }).then(saveQuery)}

15
packages/builder/src/components/integration/QueryViewerSidePanel/JSONPanel.svelte
View File

@ -1,7 +1,20 @@
<script>
export let data
export let maxRowsToDisplay = 5
$: string = JSON.stringify(data || {}, null, 2)
let string
$: {
string = JSON.stringify(data || {}, null, 2)
if (Array.isArray(data) && data.length > maxRowsToDisplay) {
string = JSON.stringify(data.slice(0, maxRowsToDisplay) || {}, null, 2)
// Display '...' at the end of the array
string = string.replace(
/(}\n])/,
`},\n ...${data.length - maxRowsToDisplay} further items\n]`
)
}
}
</script>
<textarea class="json" disabled value={string} />

30
packages/builder/src/components/integration/QueryViewerSidePanel/PreviewPanel.svelte
View File

@ -4,13 +4,17 @@
export let schema = {}
export let rows = []
export let maxRowsToDisplay = 5
$: rowsCopy = cloneDeep(rows)
let rowsToDisplay
$: rowsToDisplay = [...cloneDeep(rows).slice(0, maxRowsToDisplay)]
$: additionalRows = rows.length - maxRowsToDisplay
// Cast field in query preview response to number if specified by schema
$: {
for (let i = 0; i < rowsCopy.length; i++) {
let row = rowsCopy[i]
for (let i = 0; i < rowsToDisplay.length; i++) {
let row = rowsToDisplay[i]
for (let fieldName of Object.keys(schema)) {
if (schema[fieldName] === "number" && !isNaN(Number(row[fieldName]))) {
row[fieldName] = Number(row[fieldName])
@ -23,11 +27,27 @@
</script>
<div class="table">
<Table {schema} data={rowsCopy} allowEditing={false} />
<Table {schema} data={rowsToDisplay} allowEditing={false} />
{#if additionalRows > 0}
<div class="show-more">
...{additionalRows} further items
</div>
{/if}
</div>
<style>
.table :global(.spectrum-Table-cell) {
.table :global(.spectrum-Table-cell),
.show-more {
min-width: 100px;
}
.show-more {
display: flex;
padding: 16px;
justify-content: center;
background-color: var(--spectrum-global-color-gray-50);
border: 1px solid var(--spectrum-alias-border-color-mid);
border-top: 0;
}
</style>

2
packages/builder/src/components/integration/QueryViewerSidePanel/index.svelte
View File

@ -41,7 +41,7 @@
</div>
<div class="content">
{#if activeTab === "JSON"}
<JSONPanel data={rows[0] || {}} />
<JSONPanel data={rows?.length === 1 ? rows[0] : rows || {}} />
{:else if activeTab === "Schema"}
<SchemaPanel {onSchemaChange} {schema} />
{:else}

2
packages/pro

@ -1 +1 @@
Subproject commit aaf7101cd1493215155cc8f83124c70d53eb1be4
Subproject commit 336bf2184cf632fdc2bffbad5628e8b15dd381bd

2
packages/server/Dockerfile
View File

@ -82,6 +82,8 @@ EXPOSE 4001
# due to this causing yarn to stop installing dev dependencies
# which are actually needed to get this environment up and running
ENV NODE_ENV=production
# this is required for isolated-vm to work on Node 20+
ENV NODE_OPTIONS="--no-node-snapshot"
ENV CLUSTER_MODE=${CLUSTER_MODE}
ENV TOP_LEVEL_PATH=/app

2
packages/server/__mocks__/@sendgrid/mail.ts
View File

@ -11,7 +11,7 @@ module SendgridMock {
}
async send(msg: any) {
if (msg.to === "invalid@test.com") {
if (msg.to === "invalid@example.com") {
throw "Invalid"
}
return msg

2
packages/server/__mocks__/aws-sdk.ts
View File

@ -60,7 +60,7 @@ module AwsMock {
// @ts-ignore
this.getSignedUrl = (operation, params) => {
return `http://test.com/${params.Bucket}/${params.Key}`
return `http://example.com/${params.Bucket}/${params.Key}`
}
// @ts-ignore

6
packages/server/__mocks__/node-fetch.ts
View File

@ -36,8 +36,8 @@ module FetchMock {
if (url.includes("/api/global")) {
const user = {
email: "test@test.com",
_id: "us_test@test.com",
email: "test@example.com",
_id: "us_test@example.com",
status: "active",
roles: {},
builder: {
@ -58,7 +58,7 @@ module FetchMock {
url: "/app1",
},
})
} else if (url.includes("test.com")) {
} else if (url.includes("example.com")) {
return json({
body: opts.body,
url,

4
packages/server/nodemon.json
View File

@ -8,6 +8,6 @@
"../string-templates"
],
"ext": "js,ts,json,svelte",
"ignore": ["src/**/*.spec.ts", "src/**/*.spec.js", "../*/dist/**/*"],
"exec": "yarn build && node ./dist/index.js"
"ignore": ["**/*.spec.ts", "**/*.spec.js", "../*/dist/**/*"],
"exec": "yarn build && node --no-node-snapshot ./dist/index.js"
}

20
packages/server/package.json
View File

@ -13,8 +13,12 @@
"build": "node ./scripts/build.js",
"postbuild": "copyfiles -f ../client/dist/budibase-client.js ../client/manifest.json client && copyfiles -f ../../yarn.lock ./dist/",
"check:types": "tsc -p tsconfig.json --noEmit --paths null",
"build:isolated-vm-lib:string-templates": "esbuild --minify --bundle src/jsRunner/bundles/index-helpers.ts --outfile=src/jsRunner/bundles/index-helpers.ivm.bundle.js --platform=node --format=esm --external:handlebars",
"build:isolated-vm-lib:bson": "esbuild --minify --bundle src/jsRunner/bundles/bsonPackage.ts --outfile=src/jsRunner/bundles/bson.ivm.bundle.js --platform=node --format=esm",
"build:isolated-vm-libs": "yarn build:isolated-vm-lib:string-templates && yarn build:isolated-vm-lib:bson",
"build:dev": "yarn prebuild && tsc --build --watch --preserveWatchOutput",
"debug": "yarn build && node --expose-gc --inspect=9222 dist/index.js",
"jest": "NODE_OPTIONS=\"--no-node-snapshot $NODE_OPTIONS\" jest",
"test": "bash scripts/test.sh",
"test:memory": "jest --maxWorkers=2 --logHeapUsage --forceExit",
"test:watch": "jest --watch",
@ -49,8 +53,8 @@
"@budibase/shared-core": "0.0.0",
"@budibase/string-templates": "0.0.0",
"@budibase/types": "0.0.0",
"@bull-board/api": "3.7.0",
"@bull-board/koa": "3.9.4",
"@bull-board/api": "5.10.2",
"@bull-board/koa": "5.10.2",
"@elastic/elasticsearch": "7.10.0",
"@google-cloud/firestore": "6.8.0",
"@koa/router": "8.0.8",
@ -65,14 +69,15 @@
"cookies": "0.8.0",
"csvtojson": "2.0.10",
"curlconverter": "3.21.0",
"dd-trace": "5.0.0",
"dd-trace": "5.2.0",
"dotenv": "8.2.0",
"form-data": "4.0.0",
"global-agent": "3.0.0",
"google-auth-library": "7.12.0",
"google-spreadsheet": "3.2.0",
"ioredis": "5.3.2",
"jimp": "0.16.1",
"isolated-vm": "^4.7.2",
"jimp": "0.22.10",
"joi": "17.6.0",
"js-yaml": "4.1.0",
"jsonschema": "1.4.0",
@ -85,7 +90,7 @@
"koa2-ratelimit": "1.1.1",
"lodash": "4.17.21",
"memorystream": "0.3.1",
"mongodb": "5.7",
"mongodb": "^6.3.0",
"mssql": "10.0.1",
"mysql2": "3.5.2",
"node-fetch": "2.6.7",
@ -104,7 +109,9 @@
"svelte": "^3.49.0",
"tar": "6.1.15",
"to-json-schema": "0.2.5",
"uuid": "3.3.2",
"undici": "^6.0.1",
"undici-types": "^6.0.1",
"uuid": "^8.3.2",
"validate.js": "0.13.1",
"vm2": "^3.9.19",
"worker-farm": "1.7.0",
@ -129,6 +136,7 @@
"@types/server-destroy": "1.0.1",
"@types/supertest": "2.0.14",
"@types/tar": "6.1.5",
"@types/uuid": "8.3.4",
"apidoc": "0.50.4",
"copyfiles": "2.4.1",
"docker-compose": "0.23.17",

24
packages/server/scripts/integrations/mongo/docker-compose.yaml Normal file
View File

@ -0,0 +1,24 @@
# Use root/example as user/password credentials
version: "3.1"
services:
mongo:
image: mongo
restart: always
ports:
- 27017:27017
environment:
MONGO_INITDB_ROOT_USERNAME: root
MONGO_INITDB_ROOT_PASSWORD: example
mongo-express:
image: mongo-express
restart: always
ports:
- 8081:8081
environment:
ME_CONFIG_MONGODB_ADMINUSERNAME: root
ME_CONFIG_MONGODB_ADMINPASSWORD: example
ME_CONFIG_MONGODB_AUTH_USERNAME: admin
ME_CONFIG_MONGODB_AUTH_PASSWORD: pass
ME_CONFIG_MONGODB_URL: mongodb://root:example@mongo:27017/

4
packages/server/scripts/test.sh
View File

@ -3,12 +3,12 @@ set -e
if [[ -n $CI ]]
then
# Running in ci, where resources are limited
export NODE_OPTIONS="--max-old-space-size=4096"
export NODE_OPTIONS="--max-old-space-size=4096 --no-node-snapshot"
echo "jest --coverage --maxWorkers=2 --forceExit --workerIdleMemoryLimit=2000MB --bail $@"
jest --coverage --maxWorkers=2 --forceExit --workerIdleMemoryLimit=2000MB --bail $@
else
# --maxWorkers performs better in development
export NODE_OPTIONS="--no-node-snapshot"
echo "jest --coverage --maxWorkers=2 --forceExit $@"
jest --coverage --maxWorkers=2 --forceExit $@
fi

4
packages/server/specs/openapi.json
View File

@ -367,7 +367,7 @@
"value": {
"data": {
"_id": "us_693a73206518477283a8d5ae31103252",
"email": "test@test.com",
"email": "test@example.com",
"roles": {
"app_957b12f943d348faa61db7e18e088d0f": "BASIC"
},
@ -397,7 +397,7 @@
"data": [
{
"_id": "us_693a73206518477283a8d5ae31103252",
"email": "test@test.com",
"email": "test@example.com",
"roles": {
"app_957b12f943d348faa61db7e18e088d0f": "BASIC"
},

4
packages/server/specs/openapi.yaml
View File

@ -256,7 +256,7 @@ components:
value:
data:
_id: us_693a73206518477283a8d5ae31103252
email: test@test.com
email: test@example.com
roles:
app_957b12f943d348faa61db7e18e088d0f: BASIC
builder:
@ -278,7 +278,7 @@ components:
value:
data:
- _id: us_693a73206518477283a8d5ae31103252
email: test@test.com
email: test@example.com
roles:
app_957b12f943d348faa61db7e18e088d0f: BASIC
builder:

2
packages/server/specs/resources/user.ts
View File

@ -3,7 +3,7 @@ import Resource from "./utils/Resource"
const user = {
_id: "us_693a73206518477283a8d5ae31103252",
email: "test@test.com",
email: "test@example.com",
roles: {
app_957b12f943d348faa61db7e18e088d0f: "BASIC",
},

6
packages/server/src/api/controllers/script.ts
View File

@ -1,12 +1,12 @@
import ScriptRunner from "../../utilities/scriptRunner"
import { BBContext } from "@budibase/types"
import { Ctx } from "@budibase/types"
export async function execute(ctx: BBContext) {
export async function execute(ctx: Ctx) {
const { script, context } = ctx.request.body
const runner = new ScriptRunner(script, context)
ctx.body = runner.execute()
}
export async function save(ctx: BBContext) {
export async function save(ctx: Ctx) {
ctx.throw(501, "Not currently implemented")
}

2
packages/server/src/api/controllers/static/index.ts
View File

@ -3,7 +3,7 @@ import { InvalidFileExtensions } from "@budibase/shared-core"
require("svelte/register")
import { join } from "../../../utilities/centralPath"
import uuid from "uuid"
import * as uuid from "uuid"
import { ObjectStoreBuckets } from "../../../constants"
import { processString } from "@budibase/string-templates"
import {

2
packages/server/src/api/routes/tests/appSync.spec.ts
View File

@ -12,7 +12,7 @@ describe("/api/applications/:appId/sync", () => {
app = await config.init()
// create some users which we will use throughout the tests
await config.createUser({
email: "sync1@test.com",
email: "sync1@example.com",
roles: {
[app._id!]: roles.BUILTIN_ROLE_IDS.BASIC,
},

2
packages/server/src/api/routes/tests/datasource.spec.ts
View File

@ -77,7 +77,7 @@ describe("/datasources", () => {
const { datasource, query } = await config.dynamicVariableDatasource()
// preview once to cache variables
await preview(datasource, {
path: "www.test.com",
path: "www.example.com",
queryString: "test={{ variable3 }}",
})
// check variables in cache

6
packages/server/src/api/routes/tests/row.spec.ts
View File

@ -2031,7 +2031,7 @@ describe.each([
describe("Formula JS protection", () => {
it("should time out JS execution if a single cell takes too long", async () => {
await config.withEnv({ JS_PER_EXECUTION_TIME_LIMIT_MS: 20 }, async () => {
await config.withEnv({ JS_PER_INVOCATION_TIMEOUT_MS: 20 }, async () => {
const js = Buffer.from(
`
let i = 0;
@ -2071,8 +2071,8 @@ describe.each([
it("should time out JS execution if a multiple cells take too long", async () => {
await config.withEnv(
{
JS_PER_EXECUTION_TIME_LIMIT_MS: 20,
JS_PER_REQUEST_TIME_LIMIT_MS: 40,
JS_PER_INVOCATION_TIMEOUT_MS: 20,
JS_PER_REQUEST_TIMEOUT_MS: 40,
},
async () => {
const js = Buffer.from(

2
packages/server/src/api/routes/tests/static.spec.js
View File

@ -80,7 +80,7 @@ describe("/static", () => {
.set(config.defaultHeaders())
.expect("Content-Type", /json/)
.expect(200)
expect(res.body.signedUrl).toEqual("http://test.com/foo/bar")
expect(res.body.signedUrl).toEqual("http://example.com/foo/bar")
expect(res.body.publicUrl).toEqual(
`https://${bucket}.s3.eu-west-1.amazonaws.com/${key}`
)

2
packages/server/src/api/routes/tests/table.spec.ts
View File

@ -16,7 +16,7 @@ import {
import { checkBuilderEndpoint } from "./utilities/TestFunctions"
import * as setup from "./utilities"
import sdk from "../../../sdk"
import uuid from "uuid"
import * as uuid from "uuid"
const { basicTable } = setup.structures

2
packages/server/src/api/routes/tests/utilities/index.ts
View File

@ -9,7 +9,7 @@ function user() {
_id: "user",
_rev: "rev",
createdAt: Date.now(),
email: "test@test.com",
email: "test@example.com",
roles: {},
tenantId: "default",
status: "active",

4
packages/server/src/automations/tests/discord.spec.js
View File

@ -11,7 +11,7 @@ describe("test the outgoing webhook action", () => {
await config.init()
inputs = {
username: "joe_bloggs",
url: "http://www.test.com",
url: "http://www.example.com",
}
})
@ -19,7 +19,7 @@ describe("test the outgoing webhook action", () => {
it("should be able to run the action", async () => {
const res = await setup.runStep(setup.actions.discord.stepId, inputs)
expect(res.response.url).toEqual("http://www.test.com")
expect(res.response.url).toEqual("http://www.example.com")
expect(res.response.method).toEqual("post")
expect(res.success).toEqual(true)
})

40
packages/server/src/automations/tests/executeScript.spec.js
View File

@ -9,34 +9,40 @@ describe("test the execute script action", () => {
afterAll(setup.afterAll)
it("should be able to execute a script", async () => {
let res = await setup.runStep(
setup.actions.EXECUTE_SCRIPT.stepId,
(inputs = {
code: "return 1 + 1",
})
)
const res = await setup.runStep(setup.actions.EXECUTE_SCRIPT.stepId, {
code: "return 1 + 1",
})
expect(res.value).toEqual(2)
expect(res.success).toEqual(true)
})
it("should handle a null value", async () => {
let res = await setup.runStep(
setup.actions.EXECUTE_SCRIPT.stepId,
(inputs = {
code: null,
})
)
const res = await setup.runStep(setup.actions.EXECUTE_SCRIPT.stepId, {
code: null,
})
expect(res.response.message).toEqual("Invalid inputs")
expect(res.success).toEqual(false)
})
it("should be able to handle an error gracefully", async () => {
let res = await setup.runStep(
it("should be able to get a value from context", async () => {
const res = await setup.runStep(
setup.actions.EXECUTE_SCRIPT.stepId,
(inputs = {
code: "return something.map(x => x.name)",
})
{
code: "return steps.map(d => d.value)",
},
{
steps: [{ value: 0 }, { value: 1 }],
}
)
expect(res.value).toEqual([0, 1])
expect(res.response).toBeUndefined()
expect(res.success).toEqual(true)
})
it("should be able to handle an error gracefully", async () => {
const res = await setup.runStep(setup.actions.EXECUTE_SCRIPT.stepId, {
code: "return something.map(x => x.name)",
})
expect(res.response).toEqual("ReferenceError: something is not defined")
expect(res.success).toEqual(false)
})

10
packages/server/src/automations/tests/make.spec.ts
View File

@ -12,9 +12,9 @@ describe("test the outgoing webhook action", () => {
it("should be able to run the action", async () => {
const res = await runStep(actions.integromat.stepId, {
value1: "test",
url: "http://www.test.com",
url: "http://www.example.com",
})
expect(res.response.url).toEqual("http://www.test.com")
expect(res.response.url).toEqual("http://www.example.com")
expect(res.response.method).toEqual("post")
expect(res.success).toEqual(true)
})
@ -30,9 +30,9 @@ describe("test the outgoing webhook action", () => {
body: {
value: payload,
},
url: "http://www.test.com",
url: "http://www.example.com",
})
expect(res.response.url).toEqual("http://www.test.com")
expect(res.response.url).toEqual("http://www.example.com")
expect(res.response.method).toEqual("post")
expect(res.response.body).toEqual(payload)
expect(res.success).toEqual(true)
@ -45,7 +45,7 @@ describe("test the outgoing webhook action", () => {
body: {
value: payload,
},
url: "http://www.test.com",
url: "http://www.example.com",
})
expect(res.httpStatus).toEqual(400)
expect(res.response).toEqual("Invalid payload JSON")

4
packages/server/src/automations/tests/outgoingWebhook.spec.js
View File

@ -11,7 +11,7 @@ describe("test the outgoing webhook action", () => {
await config.init()
inputs = {
requestMethod: "POST",
url: "www.test.com",
url: "www.example.com",
requestBody: JSON.stringify({
a: 1,
}),
@ -26,7 +26,7 @@ describe("test the outgoing webhook action", () => {
inputs
)
expect(res.success).toEqual(true)
expect(res.response.url).toEqual("http://www.test.com")
expect(res.response.url).toEqual("http://www.example.com")
expect(res.response.method).toEqual("POST")
expect(JSON.parse(res.response.body).a).toEqual(1)
})

10
packages/server/src/automations/tests/sendSmtpEmail.spec.ts
View File

@ -33,7 +33,7 @@ describe("test the outgoing webhook action", () => {
jest
.spyOn(workerRequests, "sendSmtpEmail")
.mockImplementationOnce(async () =>
generateResponse("user1@test.com", "admin@test.com")
generateResponse("user1@example.com", "admin@example.com")
)
const invite = {
startTime: new Date(),
@ -43,8 +43,8 @@ describe("test the outgoing webhook action", () => {
url: "url",
}
inputs = {
to: "user1@test.com",
from: "admin@test.com",
to: "user1@example.com",
from: "admin@example.com",
subject: "hello",
contents: "testing",
cc: "cc",
@ -61,8 +61,8 @@ describe("test the outgoing webhook action", () => {
expect(res.success).toEqual(true)
expect(workerRequests.sendSmtpEmail).toHaveBeenCalledTimes(1)
expect(workerRequests.sendSmtpEmail).toHaveBeenCalledWith({
to: "user1@test.com",
from: "admin@test.com",
to: "user1@example.com",
from: "admin@example.com",
subject: "hello",
contents: "testing",
cc: "cc",

10
packages/server/src/automations/tests/zapier.spec.ts
View File

@ -12,9 +12,9 @@ describe("test the outgoing webhook action", () => {
it("should be able to run the action", async () => {
const res = await runStep(actions.zapier.stepId, {
value1: "test",
url: "http://www.test.com",
url: "http://www.example.com",
})
expect(res.response.url).toEqual("http://www.test.com")
expect(res.response.url).toEqual("http://www.example.com")
expect(res.response.method).toEqual("post")
expect(res.success).toEqual(true)
})
@ -30,9 +30,9 @@ describe("test the outgoing webhook action", () => {
body: {
value: payload,
},
url: "http://www.test.com",
url: "http://www.example.com",
})
expect(res.response.url).toEqual("http://www.test.com")
expect(res.response.url).toEqual("http://www.example.com")
expect(res.response.method).toEqual("post")
expect(res.response.body).toEqual(
`{"platform":"budibase","value1":1,"value2":2,"value3":3,"value4":4,"value5":5,"name":"Adam","age":9}`
@ -47,7 +47,7 @@ describe("test the outgoing webhook action", () => {
body: {
value: payload,
},
url: "http://www.test.com",
url: "http://www.example.com",
})
expect(res.httpStatus).toEqual(400)
expect(res.response).toEqual("Invalid payload JSON")

2
packages/server/src/db/newid.ts
View File

@ -1,4 +1,4 @@
const { v4 } = require("uuid")
import { v4 } from "uuid"
export default function (): string {
return v4().replace(/-/g, "")

6
packages/server/src/environment.ts
View File

@ -71,9 +71,9 @@ const environment = {
SELF_HOSTED: process.env.SELF_HOSTED,
HTTP_MB_LIMIT: process.env.HTTP_MB_LIMIT,
FORKED_PROCESS_NAME: process.env.FORKED_PROCESS_NAME || "main",
JS_PER_EXECUTION_TIME_LIMIT_MS:
JS_PER_INVOCATION_TIMEOUT_MS:
parseIntSafe(process.env.JS_PER_EXECUTION_TIME_LIMIT_MS) || 1000,
JS_PER_REQUEST_TIME_LIMIT_MS: parseIntSafe(
JS_PER_REQUEST_TIMEOUT_MS: parseIntSafe(
process.env.JS_PER_REQUEST_TIME_LIMIT_MS
),
// old
@ -95,6 +95,8 @@ const environment = {
TOP_LEVEL_PATH:
process.env.TOP_LEVEL_PATH || process.env.SERVER_TOP_LEVEL_PATH,
APP_MIGRATION_TIMEOUT: parseIntSafe(process.env.APP_MIGRATION_TIMEOUT),
JS_RUNNER_MEMORY_LIMIT:
parseIntSafe(process.env.JS_RUNNER_MEMORY_LIMIT) || 64,
}
// threading can cause memory issues with node-ts in development

14
packages/server/src/integrations/mongodb.ts
View File

@ -21,7 +21,6 @@ import environment from "../environment"
interface MongoDBConfig {
connectionString: string
db: string
tlsCertificateFile: string
tlsCertificateKeyFile: string
tlsCAFile: string
}
@ -320,16 +319,11 @@ const getSchema = () => {
if (environment.SELF_HOSTED) {
schema.datasource = {
...schema.datasource,
//@ts-ignore
// @ts-ignore
tls: {
type: DatasourceFieldType.FIELD_GROUP,
display: "Configure SSL",
fields: {
tlsCertificateFile: {
type: DatasourceFieldType.STRING,
required: false,
display: "Certificate file path",
},
tlsCertificateKeyFile: {
type: DatasourceFieldType.STRING,
required: false,
@ -356,7 +350,6 @@ class MongoIntegration implements IntegrationBase {
constructor(config: MongoDBConfig) {
this.config = config
const options: MongoClientOptions = {
tlsCertificateFile: config.tlsCertificateFile || undefined,
tlsCertificateKeyFile: config.tlsCertificateKeyFile || undefined,
tlsCAFile: config.tlsCAFile || undefined,
}
@ -525,7 +518,10 @@ class MongoIntegration implements IntegrationBase {
return await collection.findOneAndUpdate(
findAndUpdateJson.filter,
findAndUpdateJson.update,
findAndUpdateJson.options
{
...findAndUpdateJson.options,
includeResultMetadata: true,
}
)
}
case "count": {

1
packages/server/src/integrations/tests/mongo.spec.ts
View File

@ -221,6 +221,7 @@ describe("MongoDB Integration", () => {
})
expect(args[2]).toEqual({
upsert: false,
includeResultMetadata: true,
})
})

15
packages/server/src/jsRunner/bundles/README.md Normal file
View File

@ -0,0 +1,15 @@
# Bundles for isolated-vm
[Isolated-vm](https://github.com/laverdet/isolated-vm) requires for us to have some libraries, such as string-templates helpers, built in a single file without external dependencies. These libraries are pretty much static. To avoid building this in every dev command, in every test command and in every pipeline, these libraries are already compiled and commited into the repo.
## How are they consumed?
These libaries are compiled with a special extension: .ivm.bundle.js. This extension is configured in [esbuild](/scripts/build.js) in order to not be bundled as javascript, and to be treated as a `string` instead. This will allow us to read it's context on runtime and inject it to `isolated-vm`.
## How to update it?
These libraries are pretty much static, but they might require some updates from time to time when something changes on the source code. In order to do this, we just need to run the following command and commit the updated bundles:
```
yarn build:isolated-vm-libs
```

7
packages/server/src/jsRunner/bundles/bson.ivm.bundle.js Normal file
View File

File diff suppressed because one or more lines are too long

4
packages/server/src/jsRunner/bundles/bsonPackage.ts Normal file
View File

@ -0,0 +1,4 @@
import { EJSON } from "bson"
export { deserialize } from "bson"
export const toJson = EJSON.deserialize

59
packages/server/src/jsRunner/bundles/index-helpers.ivm.bundle.js Normal file
View File

File diff suppressed because one or more lines are too long

12
packages/server/src/jsRunner/bundles/index-helpers.ts Normal file
View File

@ -0,0 +1,12 @@
const {
getJsHelperList,
} = require("../../../../string-templates/src/helpers/list.js")
const helpers = getJsHelperList()
export default {
...helpers,
// pointing stripProtocol to a unexisting function to be able to declare it on isolated-vm
// @ts-ignore
// eslint-disable-next-line no-undef
stripProtocol: helpersStripProtocol,
}

28
packages/server/src/jsRunner/bundles/index.ts Normal file
View File

@ -0,0 +1,28 @@
import { utils } from "@budibase/shared-core"
import environment from "../../environment"
import fs from "fs"
export const enum BundleType {
HELPERS = "helpers",
BSON = "bson",
}
const bundleSourceCode = {
[BundleType.HELPERS]: "../bundles/index-helpers.ivm.bundle.js",
[BundleType.BSON]: "../bundles/bson.ivm.bundle.js",
}
export function loadBundle(type: BundleType) {
if (environment.isJest()) {
return fs.readFileSync(require.resolve(bundleSourceCode[type]), "utf-8")
}
switch (type) {
case BundleType.HELPERS:
return require("../bundles/index-helpers.ivm.bundle.js")
case BundleType.BSON:
return require("../bundles/bson.ivm.bundle.js")
default:
utils.unreachable(type)
}
}

6
packages/server/src/jsRunner.ts → packages/server/src/jsRunner/index.ts
View File

@ -1,5 +1,5 @@
import vm from "vm"
import env from "./environment"
import env from "../environment"
import { setJSRunner } from "@budibase/string-templates"
import { context, timers } from "@budibase/backend-core"
import tracer from "dd-trace"
@ -9,7 +9,7 @@ type TrackerFn = <T>(f: () => T) => T
export function init() {
setJSRunner((js: string, ctx: vm.Context) => {
return tracer.trace("runJS", {}, span => {
const perRequestLimit = env.JS_PER_REQUEST_TIME_LIMIT_MS
const perRequestLimit = env.JS_PER_REQUEST_TIMEOUT_MS
let track: TrackerFn = f => f()
if (perRequestLimit) {
const bbCtx = tracer.trace("runJS.getCurrentContext", {}, span =>
@ -53,7 +53,7 @@ export function init() {
vm.createContext(ctx)
return track(() =>
vm.runInNewContext(js, ctx, {
timeout: env.JS_PER_EXECUTION_TIME_LIMIT_MS,
timeout: env.JS_PER_INVOCATION_TIMEOUT_MS,
})
)
})

83
packages/server/src/jsRunner/tests/jsRunner.spec.ts Normal file
View File

@ -0,0 +1,83 @@
// import { validate as isValidUUID } from "uuid"
jest.mock("@budibase/handlebars-helpers/lib/math", () => {
const actual = jest.requireActual("@budibase/handlebars-helpers/lib/math")
return {
...actual,
random: () => 10,
}
})
jest.mock("@budibase/handlebars-helpers/lib/uuid", () => {
const actual = jest.requireActual("@budibase/handlebars-helpers/lib/uuid")
return {
...actual,
uuid: () => "f34ebc66-93bd-4f7c-b79b-92b5569138bc",
}
})
import { processStringSync, encodeJSBinding } from "@budibase/string-templates"
const { runJsHelpersTests } = require("@budibase/string-templates/test/utils")
import tk from "timekeeper"
import { init } from ".."
import TestConfiguration from "../../tests/utilities/TestConfiguration"
tk.freeze("2021-01-21T12:00:00")
describe("jsRunner", () => {
const config = new TestConfiguration()
beforeAll(async () => {
// Register js runner
init()
await config.init()
})
const processJS = (js: string, context?: object) => {
return config.doInContext(config.getAppId(), async () =>
processStringSync(encodeJSBinding(js), context || {})
)
}
it("it can run a basic javascript", async () => {
const output = await processJS(`return 1 + 2`)
expect(output).toBe(3)
})
// TODO This should be reenabled when running on isolated-vm
it.skip("should prevent sandbox escape", async () => {
const output = await processJS(
`return this.constructor.constructor("return process")()`
)
expect(output).toBe("Error while executing JS")
})
describe("helpers", () => {
runJsHelpersTests({
funcWrap: (func: any) => config.doInContext(config.getAppId(), func),
// testsToSkip: ["random", "uuid"],
})
// describe("uuid", () => {
// it("uuid helper returns a valid uuid", async () => {
// const result = await processJS("return helpers.uuid()")
// expect(result).toBeDefined()
// expect(isValidUUID(result)).toBe(true)
// })
// })
// describe("random", () => {
// it("random helper returns a valid number", async () => {
// const min = 1
// const max = 8
// const result = await processJS(`return helpers.random(${min}, ${max})`)
// expect(result).toBeDefined()
// expect(result).toBeGreaterThanOrEqual(min)
// expect(result).toBeLessThanOrEqual(max)
// })
// })
})
})

270
packages/server/src/jsRunner/vm/index.ts Normal file
View File

@ -0,0 +1,270 @@
import ivm from "isolated-vm"
import bson from "bson"
import url from "url"
import crypto from "crypto"
import querystring from "querystring"
import { BundleType, loadBundle } from "../bundles"
import { VM } from "@budibase/types"
class ExecutionTimeoutError extends Error {
constructor(message: string) {
super(message)
this.name = "ExecutionTimeoutError"
}
}
class ModuleHandler {
private modules: {
import: string
moduleKey: string
module: ivm.Module
}[] = []
private generateRandomKey = () => `i${crypto.randomUUID().replace(/-/g, "")}`
registerModule(module: ivm.Module, imports: string) {
this.modules.push({
moduleKey: this.generateRandomKey(),
import: imports,
module: module,
})
}
generateImports() {
return this.modules
.map(m => `import ${m.import} from "${m.moduleKey}"`)
.join(";")
}
getModule(key: string) {
const module = this.modules.find(m => m.moduleKey === key)
return module?.module
}
}
export class IsolatedVM implements VM {
private isolate: ivm.Isolate
private vm: ivm.Context
private jail: ivm.Reference
private invocationTimeout: number
private isolateAccumulatedTimeout?: number
// By default the wrapper returns itself
private codeWrapper: (code: string) => string = code => code
private moduleHandler = new ModuleHandler()
private readonly resultKey = "results"
constructor({
memoryLimit,
invocationTimeout,
isolateAccumulatedTimeout,
}: {
memoryLimit: number
invocationTimeout: number
isolateAccumulatedTimeout?: number
}) {
this.isolate = new ivm.Isolate({ memoryLimit })
this.vm = this.isolate.createContextSync()
this.jail = this.vm.global
this.jail.setSync("global", this.jail.derefInto())
this.addToContext({
[this.resultKey]: { out: "" },
})
this.invocationTimeout = invocationTimeout
this.isolateAccumulatedTimeout = isolateAccumulatedTimeout
}
withHelpers() {
const urlModule = this.registerCallbacks({
resolve: url.resolve,
parse: url.parse,
})
const querystringModule = this.registerCallbacks({
escape: querystring.escape,
})
this.addToContext({
helpersStripProtocol: new ivm.Callback((str: string) => {
var parsed = url.parse(str) as any
parsed.protocol = ""
return parsed.format()
}),
})
const injectedRequire = `const require=function req(val) {
switch (val) {
case "url": return ${urlModule};
case "querystring": return ${querystringModule};
}
}`
const helpersSource = loadBundle(BundleType.HELPERS)
const helpersModule = this.isolate.compileModuleSync(
`${injectedRequire};${helpersSource}`
)
helpersModule.instantiateSync(this.vm, specifier => {
if (specifier === "crypto") {
const cryptoModule = this.registerCallbacks({
randomUUID: crypto.randomUUID,
})
const module = this.isolate.compileModuleSync(
`export default ${cryptoModule}`
)
module.instantiateSync(this.vm, specifier => {
throw new Error(`No imports allowed. Required: ${specifier}`)
})
return module
}
throw new Error(`No imports allowed. Required: ${specifier}`)
})
this.moduleHandler.registerModule(helpersModule, "helpers")
return this
}
withContext(context: Record<string, any>) {
this.addToContext(context)
return this
}
withParsingBson(data: any) {
this.addToContext({
bsonData: bson.BSON.serialize({ data }),
})
// If we need to parse bson, we follow the next steps:
// 1. Serialise the data from potential BSON to buffer before passing it to the isolate
// 2. Deserialise the data within the isolate, to get the original data
// 3. Process script
// 4. Stringify the result in order to convert the result from BSON to json
this.codeWrapper = code =>
`(function(){
const data = deserialize(bsonData, { validation: { utf8: false } }).data;
const result = ${code}
return toJson(result);
})();`
const bsonSource = loadBundle(BundleType.BSON)
this.addToContext({
textDecoderCb: new ivm.Callback(
(args: {
constructorArgs: any
functionArgs: Parameters<InstanceType<typeof TextDecoder>["decode"]>
}) => {
const result = new TextDecoder(...args.constructorArgs).decode(
...args.functionArgs
)
return result
}
),
})
// "Polyfilling" text decoder. `bson.deserialize` requires decoding. We are creating a bridge function so we don't need to inject the full library
const textDecoderPolyfill = class TextDecoder {
constructorArgs
constructor(...constructorArgs: any) {
this.constructorArgs = constructorArgs
}
decode(...input: any) {
// @ts-ignore
return textDecoderCb({
constructorArgs: this.constructorArgs,
functionArgs: input,
})
}
}.toString()
const bsonModule = this.isolate.compileModuleSync(
`${textDecoderPolyfill};${bsonSource}`
)
bsonModule.instantiateSync(this.vm, specifier => {
throw new Error(`No imports allowed. Required: ${specifier}`)
})
this.moduleHandler.registerModule(bsonModule, "{deserialize, toJson}")
return this
}
execute(code: string): any {
if (this.isolateAccumulatedTimeout) {
const cpuMs = Number(this.isolate.cpuTime) / 1e6
if (cpuMs > this.isolateAccumulatedTimeout) {
throw new ExecutionTimeoutError(
`CPU time limit exceeded (${cpuMs}ms > ${this.isolateAccumulatedTimeout}ms)`
)
}
}
code = `${this.moduleHandler.generateImports()};results.out=${this.codeWrapper(
code
)};`
const script = this.isolate.compileModuleSync(code)
script.instantiateSync(this.vm, specifier => {
const module = this.moduleHandler.getModule(specifier)
if (module) {
return module
}
throw new Error(`"${specifier}" import not allowed`)
})
script.evaluateSync({ timeout: this.invocationTimeout })
const result = this.getFromContext(this.resultKey)
return result.out
}
private registerCallbacks(functions: Record<string, any>) {
const libId = crypto.randomUUID().replace(/-/g, "")
const x: Record<string, string> = {}
for (const [funcName, func] of Object.entries(functions)) {
const key = `f${libId}${funcName}cb`
x[funcName] = key
this.addToContext({
[key]: new ivm.Callback((...params: any[]) => (func as any)(...params)),
})
}
const mod =
`{` +
Object.entries(x)
.map(([key, func]) => `${key}: ${func}`)
.join() +
"}"
return mod
}
private addToContext(context: Record<string, any>) {
for (let key in context) {
const value = context[key]
this.jail.setSync(
key,
typeof value === "function"
? value
: new ivm.ExternalCopy(value).copyInto({ release: true })
)
}
}
private getFromContext(key: string) {
const ref = this.vm.global.getSync(key, { reference: true })
const result = ref.copySync()
ref.release()
return result
}
}

2
packages/server/src/migrations/tests/structures.ts
View File

@ -46,7 +46,7 @@ export const smtp = (conf?: SMTPConfig): SMTPConfig => {
config: {
port: 12345,
host: "smtptesthost.com",
from: "testfrom@test.com",
from: "testfrom@example.com",
subject: "Hello!",
secure: false,
...conf,

8
packages/server/src/sdk/app/applications/tests/sync.spec.ts
View File

@ -94,8 +94,8 @@ function buildRoles() {
}
describe("app user/group sync", () => {
const groupEmail = "test2@test.com",
normalEmail = "test@test.com"
const groupEmail = "test2@example.com",
normalEmail = "test@example.com"
async function checkEmail(
email: string,
opts?: { group?: boolean; notFound?: boolean }
@ -131,7 +131,7 @@ describe("app user/group sync", () => {
})
it("should be able to handle builder users", async () => {
await createUser("test3@test.com", {}, true)
await checkEmail("test3@test.com")
await createUser("test3@example.com", {}, true)
await checkEmail("test3@example.com")
})
})

17
packages/server/src/sdk/app/backups/exports.ts
View File

@ -14,6 +14,7 @@ import {
ATTACHMENT_DIRECTORY,
} from "./constants"
import fs from "fs"
import fsp from "fs/promises"
import { join } from "path"
import env from "../../../environment"
import { v4 as uuid } from "uuid"
@ -117,7 +118,7 @@ export async function exportApp(appId: string, config?: ExportOpts) {
ObjectStoreBuckets.APPS,
join(appPath, path)
)
fs.writeFileSync(join(tmpPath, path), contents)
await fsp.writeFile(join(tmpPath, path), contents)
}
}
// get all the files
@ -131,14 +132,14 @@ export async function exportApp(appId: string, config?: ExportOpts) {
const downloadedPath = join(tmpPath, appPath)
if (fs.existsSync(downloadedPath)) {
const allFiles = fs.readdirSync(downloadedPath)
const allFiles = await fsp.readdir(downloadedPath)
for (let file of allFiles) {
const path = join(downloadedPath, file)
// move out of app directory, simplify structure
fs.renameSync(path, join(downloadedPath, "..", file))
await fsp.rename(path, join(downloadedPath, "..", file))
}
// remove the old app directory created by object export
fs.rmdirSync(downloadedPath)
await fsp.rmdir(downloadedPath)
}
// enforce an export of app DB to the tmp path
const dbPath = join(tmpPath, DB_EXPORT_FILE)
@ -148,7 +149,7 @@ export async function exportApp(appId: string, config?: ExportOpts) {
})
if (config?.encryptPassword) {
for (let file of fs.readdirSync(tmpPath)) {
for (let file of await fsp.readdir(tmpPath)) {
const path = join(tmpPath, file)
// skip the attachments - too big to encrypt
@ -157,7 +158,7 @@ export async function exportApp(appId: string, config?: ExportOpts) {
{ dir: tmpPath, filename: file },
config.encryptPassword
)
fs.rmSync(path)
await fsp.rm(path)
}
}
}
@ -165,9 +166,9 @@ export async function exportApp(appId: string, config?: ExportOpts) {
// if tar requested, return where the tarball is
if (config?.tar) {
// now the tmpPath contains both the DB export and attachments, tar this
const tarPath = await tarFilesToTmp(tmpPath, fs.readdirSync(tmpPath))
const tarPath = await tarFilesToTmp(tmpPath, await fsp.readdir(tmpPath))
// cleanup the tmp export files as tarball returned
fs.rmSync(tmpPath, { recursive: true, force: true })
await fsp.rm(tmpPath, { recursive: true, force: true })
return tarPath
}

15
packages/server/src/sdk/app/backups/imports.ts
View File

@ -17,6 +17,7 @@ import { downloadTemplate } from "../../../utilities/fileSystem"
import { ObjectStoreBuckets } from "../../../constants"
import { join } from "path"
import fs from "fs"
import fsp from "fs/promises"
import sdk from "../../"
import { v4 as uuid } from "uuid"
import tar from "tar"
@ -119,7 +120,7 @@ async function getTemplateStream(template: TemplateType) {
export async function untarFile(file: { path: string }) {
const tmpPath = join(budibaseTempDir(), uuid())
fs.mkdirSync(tmpPath)
await fsp.mkdir(tmpPath)
// extract the tarball
await tar.extract({
cwd: tmpPath,
@ -130,12 +131,12 @@ export async function untarFile(file: { path: string }) {
async function decryptFiles(path: string, password: string) {
try {
for (let file of fs.readdirSync(path)) {
for (let file of await fsp.readdir(path)) {
const inputPath = join(path, file)
if (!inputPath.endsWith(ATTACHMENT_DIRECTORY)) {
const outputPath = inputPath.replace(/\.enc$/, "")
await encryption.decryptFile(inputPath, outputPath, password)
fs.rmSync(inputPath)
await fsp.rm(inputPath)
}
}
} catch (err: any) {
@ -164,14 +165,14 @@ export async function importApp(
let dbStream: any
const isTar = template.file && template?.file?.type?.endsWith("gzip")
const isDirectory =
template.file && fs.lstatSync(template.file.path).isDirectory()
template.file && (await fsp.lstat(template.file.path)).isDirectory()
let tmpPath: string | undefined = undefined
if (template.file && (isTar || isDirectory)) {
tmpPath = isTar ? await untarFile(template.file) : template.file.path
if (isTar && template.file.password) {
await decryptFiles(tmpPath, template.file.password)
}
const contents = fs.readdirSync(tmpPath)
const contents = await fsp.readdir(tmpPath)
// have to handle object import
if (contents.length && opts.importObjStoreContents) {
let promises = []
@ -182,7 +183,7 @@ export async function importApp(
continue
}
filename = join(prodAppId, filename)
if (fs.lstatSync(path).isDirectory()) {
if ((await fsp.lstat(path)).isDirectory()) {
promises.push(
objectStore.uploadDirectory(ObjectStoreBuckets.APPS, path, filename)
)
@ -211,7 +212,7 @@ export async function importApp(
await updateAutomations(prodAppId, db)
// clear up afterward
if (tmpPath) {
fs.rmSync(tmpPath, { recursive: true, force: true })
await fsp.rm(tmpPath, { recursive: true, force: true })
}
return ok
}

2
packages/server/src/sdk/app/rows/search/tests/utils.spec.ts
View File

@ -76,7 +76,7 @@ describe.each([tableWithUserCol, tableWithUsersCol])(
})
it("shouldn't change any other input", () => {
const email = "test@test.com"
const email = "test@example.com"
const params: SearchParams = {
tableId,
query: {

4
packages/server/src/threads/definitions.ts
View File

@ -1,10 +1,10 @@
import { QuerySchema, Row } from "@budibase/types"
import { Datasource, QuerySchema, Row } from "@budibase/types"
export type WorkerCallback = (error: any, response?: any) => void
export interface QueryEvent {
appId?: string
datasource: any
datasource: Datasource
queryVerb: string
fields: { [key: string]: any }
parameters: { [key: string]: any }

13
packages/server/src/threads/query.ts
View File

@ -14,13 +14,13 @@ import { context, cache, auth } from "@budibase/backend-core"
import { getGlobalIDFromUserMetadataID } from "../db/utils"
import sdk from "../sdk"
import { cloneDeep } from "lodash/fp"
import { Query } from "@budibase/types"
import { Datasource, Query, SourceName } from "@budibase/types"
import { isSQL } from "../integrations/utils"
import { interpolateSQL } from "../integrations/queries/sql"
class QueryRunner {
datasource: any
datasource: Datasource
queryVerb: string
queryId: string
fields: any
@ -68,7 +68,7 @@ class QueryRunner {
throw "Integration type does not exist."
}
if (datasourceClone.config.authConfigs) {
if (datasourceClone.config?.authConfigs) {
const updatedConfigs = []
for (let config of datasourceClone.config.authConfigs) {
updatedConfigs.push(await sdk.queries.enrichContext(config, this.ctx))
@ -93,7 +93,7 @@ class QueryRunner {
const enrichedContext = { ...enrichedParameters, ...this.ctx }
// Parse global headers
if (datasourceClone.config.defaultHeaders) {
if (datasourceClone.config?.defaultHeaders) {
datasourceClone.config.defaultHeaders = await sdk.queries.enrichContext(
datasourceClone.config.defaultHeaders,
enrichedContext
@ -152,11 +152,6 @@ class QueryRunner {
return this.execute()
}
// check for undefined response
if (!rows) {
rows = []
}
// needs to an array for next step
if (!Array.isArray(rows)) {
rows = [rows]

2
packages/server/src/utilities/fileSystem/filesystem.ts
View File

@ -4,7 +4,7 @@ import { resolve, join } from "path"
import env from "../../environment"
import tar from "tar"
const uuid = require("uuid/v4")
import { v4 as uuid } from "uuid"
export const TOP_LEVEL_PATH =
env.TOP_LEVEL_PATH || resolve(join(__dirname, "..", "..", ".."))

13
packages/string-templates/package.json
View File

@ -2,16 +2,17 @@
"name": "@budibase/string-templates",
"version": "0.0.0",
"description": "Handlebars wrapper for Budibase templating.",
"main": "src/index.cjs",
"main": "src/index.js",
"module": "dist/bundle.mjs",
"license": "MPL-2.0",
"types": "dist/index.d.ts",
"exports": {
".": {
"require": "./src/index.cjs",
"require": "./src/index.js",
"import": "./dist/bundle.mjs"
},
"./package.json": "./package.json"
"./package.json": "./package.json",
"./test/utils": "./test/utils.js"
},
"files": [
"dist",
@ -20,7 +21,7 @@
],
"scripts": {
"build": "tsc && rollup -c",
"dev": "tsc && rollup -cw",
"dev": "concurrently \"tsc --watch\" \"rollup -cw\"",
"test": "jest",
"manifest": "node ./scripts/gen-collection-info.js"
},
@ -28,12 +29,12 @@
"@budibase/handlebars-helpers": "^0.13.1",
"dayjs": "^1.10.8",
"handlebars": "^4.7.6",
"lodash.clonedeep": "^4.5.0",
"vm2": "^3.9.19"
"lodash.clonedeep": "^4.5.0"
},
"devDependencies": {
"@rollup/plugin-commonjs": "^17.1.0",
"@rollup/plugin-json": "^4.1.0",
"concurrently": "^8.2.2",
"doctrine": "^3.0.0",
"jest": "29.7.0",
"marked": "^4.0.10",

11
packages/string-templates/src/errors.js Normal file
View File

@ -0,0 +1,11 @@
class JsErrorTimeout extends Error {
code = "ERR_SCRIPT_EXECUTION_TIMEOUT"
constructor() {
super()
}
}
module.exports = {
JsErrorTimeout,
}

4
packages/string-templates/src/helpers/javascript.js
View File

@ -4,7 +4,7 @@ const { LITERAL_MARKER } = require("../helpers/constants")
const { getJsHelperList } = require("./list")
// The method of executing JS scripts depends on the bundle being built.
// This setter is used in the entrypoint (either index.cjs or index.mjs).
// This setter is used in the entrypoint (either index.js or index.mjs).
let runJS
module.exports.setJSRunner = runner => (runJS = runner)
@ -42,7 +42,7 @@ module.exports.processJS = (handlebars, context) => {
try {
// Wrap JS in a function and immediately invoke it.
// This is required to allow the final `return` statement to be valid.
const js = `function run(){${atob(handlebars)}};run();`
const js = `(function(){${atob(handlebars)}})();`
// Our $ context function gets a value from context.
// We clone the context to avoid mutation in the binding affecting real

33
packages/string-templates/src/helpers/list.js
View File

@ -1,29 +1,42 @@
const externalHandlebars = require("./external")
const helperList = require("@budibase/handlebars-helpers")
const { date, duration } = require("./date")
let helpers = undefined
// https://github.com/evanw/esbuild/issues/56
const externalCollections = {
math: require("@budibase/handlebars-helpers/lib/math"),
array: require("@budibase/handlebars-helpers/lib/array"),
number: require("@budibase/handlebars-helpers/lib/number"),
url: require("@budibase/handlebars-helpers/lib/url"),
string: require("@budibase/handlebars-helpers/lib/string"),
comparison: require("@budibase/handlebars-helpers/lib/comparison"),
object: require("@budibase/handlebars-helpers/lib/object"),
regex: require("@budibase/handlebars-helpers/lib/regex"),
uuid: require("@budibase/handlebars-helpers/lib/uuid"),
}
const helpersToRemoveForJs = ["sortBy"]
module.exports.helpersToRemoveForJs = helpersToRemoveForJs
const addedHelpers = {
date: date,
duration: duration,
}
let helpers = undefined
module.exports.getJsHelperList = () => {
if (helpers) {
return helpers
}
helpers = {}
let constructed = []
for (let collection of externalHandlebars.externalCollections) {
constructed.push(helperList[collection]())
}
for (let collection of constructed) {
for (let collection of Object.values(externalCollections)) {
for (let [key, func] of Object.entries(collection)) {
// Handlebars injects the hbs options to the helpers by default. We are adding an empty {} as a last parameter to simulate it
helpers[key] = (...props) => func(...props, {})
}
}
for (let key of Object.keys(externalHandlebars.addedHelpers)) {
helpers[key] = externalHandlebars.addedHelpers[key]
for (let key of Object.keys(addedHelpers)) {
helpers[key] = addedHelpers[key]
}
for (const toRemove of helpersToRemoveForJs) {

38
packages/string-templates/src/index.cjs
View File

@ -1,38 +0,0 @@
const templates = require("./index.js")
/**
* CJS entrypoint for rollup
*/
module.exports.isValid = templates.isValid
module.exports.makePropSafe = templates.makePropSafe
module.exports.getManifest = templates.getManifest
module.exports.isJSBinding = templates.isJSBinding
module.exports.encodeJSBinding = templates.encodeJSBinding
module.exports.decodeJSBinding = templates.decodeJSBinding
module.exports.processStringSync = templates.processStringSync
module.exports.processObjectSync = templates.processObjectSync
module.exports.processString = templates.processString
module.exports.processObject = templates.processObject
module.exports.doesContainStrings = templates.doesContainStrings
module.exports.doesContainString = templates.doesContainString
module.exports.disableEscaping = templates.disableEscaping
module.exports.findHBSBlocks = templates.findHBSBlocks
module.exports.convertToJS = templates.convertToJS
module.exports.setJSRunner = templates.setJSRunner
module.exports.FIND_ANY_HBS_REGEX = templates.FIND_ANY_HBS_REGEX
module.exports.helpersToRemoveForJs = templates.helpersToRemoveForJs
if (!process.env.NO_JS) {
const { VM } = require("vm2")
const { setJSRunner } = require("./helpers/javascript")
/**
* Use vm2 to run JS scripts in a node env
*/
setJSRunner((js, context) => {
const vm = new VM({
sandbox: context,
timeout: 1000,
})
return vm.run(js)
})
}

5
packages/string-templates/src/index.js
View File

@ -395,4 +395,9 @@ module.exports.convertToJS = hbs => {
}
module.exports.FIND_ANY_HBS_REGEX = FIND_ANY_HBS_REGEX
const errors = require("./errors")
// We cannot use dynamic exports, otherwise the typescript file will not be generating it
module.exports.JsErrorTimeout = errors.JsErrorTimeout
module.exports.helpersToRemoveForJs = helpersToRemoveForJs

2
packages/string-templates/src/index.mjs
View File

@ -38,3 +38,5 @@ if (process && !process.env.NO_JS) {
return vm.runInNewContext(js, context, { timeout: 1000 })
})
}
export * from "./errors.js"

2
packages/string-templates/test/basic.spec.js
View File

@ -8,7 +8,7 @@ const {
doesContainString,
disableEscaping,
findHBSBlocks,
} = require("../src/index.cjs")
} = require("../src/index.js")
describe("Test that the string processing works correctly", () => {
it("should process a basic template string", async () => {

2
packages/string-templates/test/escapes.spec.js
View File

@ -1,4 +1,4 @@
const { processString } = require("../src/index.cjs")
const { processString } = require("../src/index.js")
describe("Handling context properties with spaces in their name", () => {
it("should allow through literal specifiers", async () => {

2
packages/string-templates/test/examples/table.json
View File

@ -2,7 +2,7 @@
"user":{
"_id":"ro_ta_users_us_b0bc7ba0ce304294accc1ced8165dd23",
"_rev":"1-e9199d92e7286005a9c11c614fdbcc51",
"email":"test2@test.com",
"email":"test2@example.com",
"status":"active",
"roleId":"PUBLIC",
"test-Created By_text":"",

2
packages/string-templates/test/hbsToJs.spec.js
View File

@ -1,4 +1,4 @@
const { convertToJS } = require("../src/index.cjs")
const { convertToJS } = require("../src/index.js")
function checkLines(response, lines) {
const toCheck = response.split("\n")

4
packages/string-templates/test/helpers.spec.js
View File

@ -1,4 +1,4 @@
const { processString, processObject, isValid } = require("../src/index.cjs")
const { processString, processObject, isValid } = require("../src/index.js")
const tableJson = require("./examples/table.json")
const dayjs = require("dayjs")
const { UUID_REGEX } = require("./constants")
@ -448,7 +448,7 @@ describe("Cover a few complex use cases", () => {
it("getting a nice date from the user", async () => {
const input = { text: `{{ date user.subscriptionDue "DD-MM" }}` }
const context = JSON.parse(
`{"user":{"email":"test@test.com","roleId":"ADMIN","type":"user","tableId":"ta_users","subscriptionDue":"2021-01-12T12:00:00.000Z","_id":"ro_ta_users_us_test@test.com","_rev":"2-24cc794985eb54183ecb93e148563f3d"}}`
`{"user":{"email":"test@example.com","roleId":"ADMIN","type":"user","tableId":"ta_users","subscriptionDue":"2021-01-12T12:00:00.000Z","_id":"ro_ta_users_us_test@example.com","_rev":"2-24cc794985eb54183ecb93e148563f3d"}}`
)
const output = await processObject(input, context)
expect(output.text).toBe("12-01")

261
packages/string-templates/test/javascript.spec.js
View File

@ -1,149 +1,156 @@
const { processStringSync, encodeJSBinding } = require("../src/index.cjs")
const vm = require("vm")
const {
processStringSync,
encodeJSBinding,
setJSRunner,
} = require("../src/index.js")
const { UUID_REGEX } = require("./constants")
const processJS = (js, context) => {
return processStringSync(encodeJSBinding(js), context)
}
describe("Test the JavaScript helper", () => {
it("should execute a simple expression", () => {
const output = processJS(`return 1 + 2`)
expect(output).toBe(3)
})
it("should be able to use primitive bindings", () => {
const output = processJS(`return $("foo")`, {
foo: "bar",
describe("Javascript", () => {
beforeAll(() => {
setJSRunner((js, context) => {
return vm.runInNewContext(js, context, { timeout: 1000 })
})
expect(output).toBe("bar")
})
it("should be able to use an object binding", () => {
const output = processJS(`return $("foo").bar`, {
foo: {
bar: "baz",
},
describe("Test the JavaScript helper", () => {
it("should execute a simple expression", () => {
const output = processJS(`return 1 + 2`)
expect(output).toBe(3)
})
expect(output).toBe("baz")
})
it("should be able to use a complex object binding", () => {
const output = processJS(`return $("foo").bar[0].baz`, {
foo: {
bar: [
{
baz: "shazbat",
},
],
},
it("should be able to use primitive bindings", () => {
const output = processJS(`return $("foo")`, {
foo: "bar",
})
expect(output).toBe("bar")
})
expect(output).toBe("shazbat")
})
it("should be able to use a deep binding", () => {
const output = processJS(`return $("foo.bar.baz")`, {
foo: {
bar: {
baz: "shazbat",
},
},
})
expect(output).toBe("shazbat")
})
it("should be able to return an object", () => {
const output = processJS(`return $("foo")`, {
foo: {
bar: {
baz: "shazbat",
},
},
})
expect(output.bar.baz).toBe("shazbat")
})
it("should be able to return an array", () => {
const output = processJS(`return $("foo")`, {
foo: ["a", "b", "c"],
})
expect(output[2]).toBe("c")
})
it("should be able to return null", () => {
const output = processJS(`return $("foo")`, {
foo: null,
})
expect(output).toBe(null)
})
it("should be able to return undefined", () => {
const output = processJS(`return $("foo")`, {
foo: undefined,
})
expect(output).toBe(undefined)
})
it("should be able to return 0", () => {
const output = processJS(`return $("foo")`, {
foo: 0,
})
expect(output).toBe(0)
})
it("should be able to return an empty string", () => {
const output = processJS(`return $("foo")`, {
foo: "",
})
expect(output).toBe("")
})
it("should be able to use a deep array binding", () => {
const output = processJS(`return $("foo.0.bar")`, {
foo: [
{
it("should be able to use an object binding", () => {
const output = processJS(`return $("foo").bar`, {
foo: {
bar: "baz",
},
],
})
expect(output).toBe("baz")
})
it("should be able to use a complex object binding", () => {
const output = processJS(`return $("foo").bar[0].baz`, {
foo: {
bar: [
{
baz: "shazbat",
},
],
},
})
expect(output).toBe("shazbat")
})
it("should be able to use a deep binding", () => {
const output = processJS(`return $("foo.bar.baz")`, {
foo: {
bar: {
baz: "shazbat",
},
},
})
expect(output).toBe("shazbat")
})
it("should be able to return an object", () => {
const output = processJS(`return $("foo")`, {
foo: {
bar: {
baz: "shazbat",
},
},
})
expect(output.bar.baz).toBe("shazbat")
})
it("should be able to return an array", () => {
const output = processJS(`return $("foo")`, {
foo: ["a", "b", "c"],
})
expect(output[2]).toBe("c")
})
it("should be able to return null", () => {
const output = processJS(`return $("foo")`, {
foo: null,
})
expect(output).toBe(null)
})
it("should be able to return undefined", () => {
const output = processJS(`return $("foo")`, {
foo: undefined,
})
expect(output).toBe(undefined)
})
it("should be able to return 0", () => {
const output = processJS(`return $("foo")`, {
foo: 0,
})
expect(output).toBe(0)
})
it("should be able to return an empty string", () => {
const output = processJS(`return $("foo")`, {
foo: "",
})
expect(output).toBe("")
})
it("should be able to use a deep array binding", () => {
const output = processJS(`return $("foo.0.bar")`, {
foo: [
{
bar: "baz",
},
],
})
expect(output).toBe("baz")
})
it("should handle errors", () => {
const output = processJS(`throw "Error"`)
expect(output).toBe("Error while executing JS")
})
it("should timeout after one second", () => {
const output = processJS(`while (true) {}`)
expect(output).toBe("Timed out while executing JS")
})
it("should prevent access to the process global", () => {
const output = processJS(`return process`)
expect(output).toBe("Error while executing JS")
})
expect(output).toBe("baz")
})
it("should handle errors", () => {
const output = processJS(`throw "Error"`)
expect(output).toBe("Error while executing JS")
})
describe("check JS helpers", () => {
it("should error if using the format helper. not helpers.", () => {
const output = processJS(`return helper.toInt(4.3)`)
expect(output).toBe("Error while executing JS")
})
it("should timeout after one second", () => {
const output = processJS(`while (true) {}`)
expect(output).toBe("Timed out while executing JS")
})
it("should be able to use toInt", () => {
const output = processJS(`return helpers.toInt(4.3)`)
expect(output).toBe(4)
})
it("should prevent access to the process global", () => {
const output = processJS(`return process`)
expect(output).toBe("Error while executing JS")
})
it("should prevent sandbox escape", () => {
const output = processJS(
`return this.constructor.constructor("return process")()`
)
expect(output).toBe("Error while executing JS")
})
})
describe("check JS helpers", () => {
it("should error if using the format helper. not helpers.", () => {
const output = processJS(`return helper.toInt(4.3)`)
expect(output).toBe("Error while executing JS")
})
it("should be able to use toInt", () => {
const output = processJS(`return helpers.toInt(4.3)`)
expect(output).toBe(4)
})
it("should be able to use uuid", () => {
const output = processJS(`return helpers.uuid()`)
expect(output).toMatch(UUID_REGEX)
it("should be able to use uuid", () => {
const output = processJS(`return helpers.uuid()`)
expect(output).toMatch(UUID_REGEX)
})
})
})

111
packages/string-templates/test/manifest.spec.js
View File

@ -1,3 +1,5 @@
const vm = require("vm")
jest.mock("@budibase/handlebars-helpers/lib/math", () => {
const actual = jest.requireActual("@budibase/handlebars-helpers/lib/math")
@ -15,81 +17,29 @@ jest.mock("@budibase/handlebars-helpers/lib/uuid", () => {
}
})
const fs = require("fs")
const {
processString,
convertToJS,
processStringSync,
encodeJSBinding,
} = require("../src/index.cjs")
const { processString, setJSRunner } = require("../src/index.js")
const tk = require("timekeeper")
const { getJsHelperList } = require("../src/helpers")
const { getParsedManifest, runJsHelpersTests } = require("./utils")
tk.freeze("2021-01-21T12:00:00")
const processJS = (js, context) => {
return processStringSync(encodeJSBinding(js), context)
}
const manifest = JSON.parse(
fs.readFileSync(require.resolve("../manifest.json"), "utf8")
)
const collections = Object.keys(manifest)
const examples = collections.reduce((acc, collection) => {
const functions = Object.entries(manifest[collection])
.filter(([_, details]) => details.example)
.map(([name, details]) => {
const example = details.example
let [hbs, js] = example.split("->").map(x => x.trim())
if (!js) {
// The function has no return value
return
}
// Trim 's
js = js.replace(/^\'|\'$/g, "")
if ((parsedExpected = tryParseJson(js))) {
if (Array.isArray(parsedExpected)) {
if (typeof parsedExpected[0] === "object") {
js = JSON.stringify(parsedExpected)
} else {
js = parsedExpected.join(",")
}
}
}
const requiresHbsBody = details.requiresBlock
return [name, { hbs, js, requiresHbsBody }]
})
.filter(x => !!x)
if (Object.keys(functions).length) {
acc[collection] = functions
}
return acc
}, {})
function escapeRegExp(string) {
return string.replace(/[.*+?^${}()|[\]\\]/g, "\\$&") // $& means the whole matched string
}
function tryParseJson(str) {
if (typeof str !== "string") {
return
}
try {
return JSON.parse(str.replace(/\'/g, '"'))
} catch (e) {
return
}
}
describe("manifest", () => {
const manifest = getParsedManifest()
beforeAll(() => {
setJSRunner((js, context) => {
return vm.runInNewContext(js, context, { timeout: 1000 })
})
})
describe("examples are valid", () => {
describe.each(Object.keys(examples))("%s", collection => {
it.each(examples[collection])("%s", async (_, { hbs, js }) => {
describe.each(Object.keys(manifest))("%s", collection => {
it.each(manifest[collection])("%s", async (_, { hbs, js }) => {
const context = {
double: i => i * 2,
isString: x => typeof x === "string",
@ -108,36 +58,5 @@ describe("manifest", () => {
})
})
describe("can be parsed and run as js", () => {
const jsHelpers = getJsHelperList()
const jsExamples = Object.keys(examples).reduce((acc, v) => {
acc[v] = examples[v].filter(([key]) => jsHelpers[key])
return acc
}, {})
describe.each(Object.keys(jsExamples))("%s", collection => {
it.each(
jsExamples[collection].filter(
([_, { requiresHbsBody }]) => !requiresHbsBody
)
)("%s", async (_, { hbs, js }) => {
const context = {
double: i => i * 2,
isString: x => typeof x === "string",
}
const arrays = hbs.match(/\[[^/\]]+\]/)
arrays?.forEach((arrayString, i) => {
hbs = hbs.replace(new RegExp(escapeRegExp(arrayString)), `array${i}`)
context[`array${i}`] = JSON.parse(arrayString.replace(/\'/g, '"'))
})
let convertedJs = convertToJS(hbs)
let result = processJS(convertedJs, context)
result = result.replace(/&nbsp;/g, " ")
expect(result).toEqual(js)
})
})
})
runJsHelpersTests()
})

2
packages/string-templates/test/renderApp.spec.js
View File

@ -1,4 +1,4 @@
const { processString } = require("../src/index.cjs")
const { processString } = require("../src/index.js")
describe("specific test case for whether or not full app template can still be rendered", () => {
it("should be able to render the app template", async () => {

111
packages/string-templates/test/utils.js Normal file
View File

@ -0,0 +1,111 @@
const { getManifest } = require("../src")
const { getJsHelperList } = require("../src/helpers")
const {
convertToJS,
processStringSync,
encodeJSBinding,
} = require("../src/index.js")
function tryParseJson(str) {
if (typeof str !== "string") {
return
}
try {
return JSON.parse(str.replace(/'/g, '"'))
} catch (e) {
return
}
}
const getParsedManifest = () => {
const manifest = getManifest()
const collections = Object.keys(manifest)
const examples = collections.reduce((acc, collection) => {
const functions = Object.entries(manifest[collection])
.filter(([_, details]) => details.example)
.map(([name, details]) => {
const example = details.example
let [hbs, js] = example.split("->").map(x => x.trim())
if (!js) {
// The function has no return value
return
}
// Trim 's
js = js.replace(/^'|'$/g, "")
let parsedExpected
if ((parsedExpected = tryParseJson(js))) {
if (Array.isArray(parsedExpected)) {
if (typeof parsedExpected[0] === "object") {
js = JSON.stringify(parsedExpected)
} else {
js = parsedExpected.join(",")
}
}
}
const requiresHbsBody = details.requiresBlock
return [name, { hbs, js, requiresHbsBody }]
})
.filter(x => !!x)
if (Object.keys(functions).length) {
acc[collection] = functions
}
return acc
}, {})
return examples
}
module.exports.getParsedManifest = getParsedManifest
module.exports.runJsHelpersTests = ({ funcWrap, testsToSkip } = {}) => {
funcWrap = funcWrap || (delegate => delegate())
const manifest = getParsedManifest()
const processJS = (js, context) => {
return funcWrap(() => processStringSync(encodeJSBinding(js), context))
}
function escapeRegExp(string) {
return string.replace(/[.*+?^${}()|[\]\\]/g, "\\$&") // $& means the whole matched string
}
describe("can be parsed and run as js", () => {
const jsHelpers = getJsHelperList()
const jsExamples = Object.keys(manifest).reduce((acc, v) => {
acc[v] = manifest[v].filter(([key]) => jsHelpers[key])
return acc
}, {})
describe.each(Object.keys(jsExamples))("%s", collection => {
const examplesToRun = jsExamples[collection]
.filter(([_, { requiresHbsBody }]) => !requiresHbsBody)
.filter(([key]) => !testsToSkip?.includes(key))
examplesToRun.length &&
it.each(examplesToRun)("%s", async (_, { hbs, js }) => {
const context = {
double: i => i * 2,
isString: x => typeof x === "string",
}
const arrays = hbs.match(/\[[^/\]]+\]/)
arrays?.forEach((arrayString, i) => {
hbs = hbs.replace(
new RegExp(escapeRegExp(arrayString)),
`array${i}`
)
context[`array${i}`] = JSON.parse(arrayString.replace(/'/g, '"'))
})
let convertedJs = convertToJS(hbs)
let result = await processJS(convertedJs, context)
result = result.replace(/&nbsp;/g, " ")
expect(result).toEqual(js)
})
})
})
}

1
packages/types/src/sdk/index.ts
View File

@ -20,3 +20,4 @@ export * from "./cli"
export * from "./websocket"
export * from "./permissions"
export * from "./row"
export * from "./vm"

3
packages/types/src/sdk/vm.ts Normal file
View File

@ -0,0 +1,3 @@
export interface VM {
execute(code: string): any
}

2
packages/worker/Dockerfile
View File

@ -44,6 +44,8 @@ EXPOSE 4001
# due to this causing yarn to stop installing dev dependencies
# which are actually needed to get this environment up and running
ENV NODE_ENV=production
# this is required for isolated-vm to work on Node 20+
ENV NODE_OPTIONS="--no-node-snapshot"
ENV CLUSTER_MODE=${CLUSTER_MODE}
ENV SERVICE=worker-service
ENV POSTHOG_TOKEN=phc_bIjZL7oh2GEUd2vqvTBH8WvrX0fWTFQMs6H5KQxiUxU

4
packages/worker/nodemon.json
View File

@ -8,6 +8,6 @@
"../string-templates"
],
"ext": "js,ts,json",
"ignore": ["src/**/*.spec.ts", "src/**/*.spec.js", "../*/dist/**/*"],
"exec": "yarn build && node dist/index.js"
"ignore": ["**/*.spec.ts", "**/*.spec.js", "../*/dist/**/*"],
"exec": "yarn build && node --no-node-snapshot dist/index.js"
}

6
packages/worker/package.json
View File

@ -48,7 +48,7 @@
"bcrypt": "5.1.0",
"bcryptjs": "2.4.3",
"bull": "4.10.1",
"dd-trace": "5.0.0",
"dd-trace": "5.2.0",
"dotenv": "8.6.0",
"global-agent": "3.0.0",
"ical-generator": "4.1.0",
@ -68,7 +68,9 @@
"passport-local": "1.0.0",
"pouchdb": "7.3.0",
"pouchdb-all-dbs": "1.1.1",
"server-destroy": "1.0.1"
"server-destroy": "1.0.1",
"undici": "^6.0.1",
"undici-types": "^6.0.1"
},
"devDependencies": {
"@swc/core": "1.3.71",

12
packages/worker/scripts/load/heavyUse.js
View File

@ -23,27 +23,27 @@ const USERS = [
password: "test",
},
{
email: "loadtest2@test.com",
email: "loadtest2@example.com",
password: "test",
},
{
email: "loadtest3@test.com",
email: "loadtest3@example.com",
password: "test",
},
{
email: "loadtest4@test.com",
email: "loadtest4@example.com",
password: "test",
},
{
email: "loadtest5@test.com",
email: "loadtest5@example.com",
password: "test",
},
{
email: "loadtest6@test.com",
email: "loadtest6@example.com",
password: "test",
},
{
email: "loadtest7@test.com",
email: "loadtest7@example.com",
password: "test",
},
]

2
packages/worker/src/api/routes/global/tests/auth.spec.ts
View File

@ -80,7 +80,7 @@ describe("/api/global/auth", () => {
it("should return 403 when user doesn't exist", async () => {
const tenantId = config.tenantId!
const email = "invaliduser@test.com"
const email = "invaliduser@example.com"
const password = "password"
const response = await config.api.auth.login(

2
packages/worker/src/api/routes/global/tests/users.spec.ts
View File

@ -490,7 +490,7 @@ describe("/api/global/users", () => {
it("should not be able to update email address", async () => {
const email = structures.email()
const user = await config.createUser(structures.users.user({ email }))
user.email = "new@test.com"
user.email = "new@example.com"
const response = await config.api.users.saveUser(user, 400)

2
packages/worker/src/tests/api/email.ts
View File

@ -10,7 +10,7 @@ export class EmailAPI extends TestAPI {
return this.request
.post(`/api/global/email/send`)
.send({
email: "test@test.com",
email: "test@example.com",
purpose,
tenantId: this.config.getTenantId(),
userId: this.config.user?._id!,

4
packages/worker/src/tests/structures/configs.ts
View File

@ -45,7 +45,7 @@ export function smtp(conf?: any): SMTPConfig {
config: {
port: 12345,
host: "smtptesthost.com",
from: "testfrom@test.com",
from: "testfrom@example.com",
subject: "Hello!",
secure: false,
...conf,
@ -59,7 +59,7 @@ export function smtpEthereal(): SMTPConfig {
config: {
port: 587,
host: "smtp.ethereal.email",
from: "testfrom@test.com",
from: "testfrom@example.com",
secure: false,
auth: {
user: "wyatt.zulauf29@ethereal.email",

2
scripts/build.js
View File

@ -49,6 +49,7 @@ function runBuild(entry, outfile) {
preserveSymlinks: true,
loader: {
".svelte": "copy",
".ivm.bundle.js": "text",
},
metafile: true,
external: [
@ -60,6 +61,7 @@ function runBuild(entry, outfile) {
"bcrypt",
"bcryptjs",
"graphql/*",
"bson",
],
}

1547
yarn.lock
View File

File diff suppressed because it is too large Load Diff