Merge remote-tracking branch 'origin/master' into poc/generate-tables-using-ai

hosting/proxy/Dockerfile

@@ -17,6 +17,7 @@ COPY error.html /usr/share/nginx/html/error.html
 # Default environment
 ENV PROXY_RATE_LIMIT_WEBHOOKS_PER_SECOND=10
 ENV PROXY_RATE_LIMIT_API_PER_SECOND=20
+ENV PROXY_TIMEOUT_SECONDS=120
 # Use docker-compose values as defaults for backwards compatibility
 ENV APPS_UPSTREAM_URL=http://app-service:4002
 ENV WORKER_UPSTREAM_URL=http://worker-service:4003

hosting/proxy/nginx.prod.conf

@@ -144,9 +144,9 @@ http {
       limit_req zone=ratelimit burst=20 nodelay;
 
       # 120s timeout on API requests
-      proxy_read_timeout 120s;
-      proxy_connect_timeout 120s;
-      proxy_send_timeout 120s;
+      proxy_read_timeout ${PROXY_TIMEOUT_SECONDS}s;
+      proxy_connect_timeout ${PROXY_TIMEOUT_SECONDS}s;
+      proxy_send_timeout ${PROXY_TIMEOUT_SECONDS}s;
 
       proxy_http_version  1.1;
       proxy_set_header    Connection          $connection_upgrade;
@@ -164,9 +164,9 @@ http {
 
       # Rest of configuration copied from /api/ location above
       # 120s timeout on API requests
-      proxy_read_timeout 120s;
-      proxy_connect_timeout 120s;
-      proxy_send_timeout 120s;
+      proxy_read_timeout ${PROXY_TIMEOUT_SECONDS}s;
+      proxy_connect_timeout ${PROXY_TIMEOUT_SECONDS}s;
+      proxy_send_timeout ${PROXY_TIMEOUT_SECONDS}s;
 
       proxy_http_version  1.1;
       proxy_set_header    Connection          $connection_upgrade;

lerna.json

@@ -1,6 +1,6 @@
 {
   "$schema": "node_modules/lerna/schemas/lerna-schema.json",
-  "version": "3.8.5",
+  "version": "3.8.6",
   "npmClient": "yarn",
   "concurrency": 20,
   "command": {

packages/backend-core/package.json

@@ -52,6 +52,7 @@
     "joi": "17.6.0",
     "jsonwebtoken": "9.0.2",
     "knex": "2.4.2",
+    "koa": "2.15.4",
     "koa-passport": "^6.0.0",
     "koa-pino-logger": "4.0.0",
     "lodash": "4.17.21",

packages/backend-core/src/context/mainContext.ts

@@ -10,7 +10,14 @@ import {
   StaticDatabases,
   DEFAULT_TENANT_ID,
 } from "../constants"
-import { Database, IdentityContext, Snippet, App, Table } from "@budibase/types"
+import {
+  Database,
+  IdentityContext,
+  Snippet,
+  App,
+  Table,
+  License,
+} from "@budibase/types"
 import { ContextMap } from "./types"
 
 let TEST_APP_ID: string | null = null
@@ -171,6 +178,18 @@ export async function doInSelfHostTenantUsingCloud<T>(
   return newContext(updates, task)
 }
 
+export async function doInLicenseContext<T>(
+  license: License,
+  task: () => T
+): Promise<T> {
+  return newContext({ license }, task)
+}
+
+export function getLicense(): License | undefined {
+  const context = Context.get()
+  return context?.license
+}
+
 export function isSelfHostUsingCloud() {
   const context = Context.get()
   return !!context?.isSelfHostUsingCloud

packages/backend-core/src/context/types.ts

@@ -1,4 +1,4 @@
-import { IdentityContext, Snippet, Table, VM } from "@budibase/types"
+import { IdentityContext, License, Snippet, Table, VM } from "@budibase/types"
 import { OAuth2Client } from "google-auth-library"
 import { GoogleSpreadsheet } from "google-spreadsheet"
 
@@ -6,6 +6,7 @@ import { GoogleSpreadsheet } from "google-spreadsheet"
 export type ContextMap = {
   tenantId?: string
   isSelfHostUsingCloud?: boolean
+  license?: License
   appId?: string
   identity?: IdentityContext
   environmentVariables?: Record<string, string>

packages/backend-core/src/middleware/contentSecurityPolicy.ts

@@ -1,4 +1,7 @@
 import crypto from "crypto"
+import { app } from "../cache"
+import { Feature, Ctx } from "@budibase/types"
+import { Middleware, Next } from "koa"
 
 const CSP_DIRECTIVES = {
   "default-src": ["'self'"],
@@ -86,28 +89,46 @@ const CSP_DIRECTIVES = {
   "worker-src": ["blob:", "'self'"],
 }
 
-export async function contentSecurityPolicy(ctx: any, next: any) {
-  try {
-    const nonce = crypto.randomBytes(16).toString("base64")
+const contentSecurityPolicy = (async (ctx: Ctx, next: Next) => {
+  const nonce = crypto.randomBytes(16).toString("base64")
+  ctx.state.nonce = nonce
+  let directives = { ...CSP_DIRECTIVES }
+  directives["script-src"] = [
+    ...CSP_DIRECTIVES["script-src"],
+    `'nonce-${nonce}'`,
+  ]
 
-    const directives = { ...CSP_DIRECTIVES }
-    directives["script-src"] = [
-      ...CSP_DIRECTIVES["script-src"],
-      `'nonce-${nonce}'`,
-    ]
-
-    ctx.state.nonce = nonce
-
-    const cspHeader = Object.entries(directives)
-      .map(([key, sources]) => `${key} ${sources.join(" ")}`)
-      .join("; ")
-    ctx.set("Content-Security-Policy", cspHeader)
-    await next()
-  } catch (err: any) {
-    console.error(
-      `Error occurred in Content-Security-Policy middleware: ${err}`
-    )
+  // Add custom app CSP whitelist
+  const licensed = ctx.user?.license?.features.includes(
+    Feature.CUSTOM_APP_SCRIPTS
+  )
+  if (licensed && ctx.appId) {
+    try {
+      const appMetadata = await app.getAppMetadata(ctx.appId)
+      if ("name" in appMetadata) {
+        for (let script of appMetadata.scripts || []) {
+          const inclusions = (script.cspWhitelist || "")
+            .split(",")
+            .filter(url => !!url?.trim().length)
+          directives["default-src"] = [
+            ...directives["default-src"],
+            ...inclusions,
+          ]
+        }
+      }
+    } catch (err) {
+      // Log an error but always proceed using the default CSP
+      console.error(
+        `Error occurred in Content-Security-Policy middleware: ${err}`
+      )
+    }
   }
-}
+
+  const cspHeader = Object.entries(directives)
+    .map(([key, sources]) => `${key} ${sources.join(" ")}`)
+    .join("; ")
+  ctx.set("Content-Security-Policy", cspHeader)
+  await next()
+}) as Middleware
 
 export default contentSecurityPolicy

packages/backend-core/src/middleware/tests/contentSecurityPolicy.spec.ts

@@ -1,10 +1,18 @@
 import crypto from "crypto"
 import contentSecurityPolicy from "../contentSecurityPolicy"
+import { app } from "../../cache"
+import { Feature, App } from "@budibase/types"
+import { users, licenses } from "../../../tests/core/utilities/structures"
 
 jest.mock("crypto", () => ({
   randomBytes: jest.fn(),
   randomUUID: jest.fn(),
 }))
+jest.mock("../../cache", () => ({
+  app: {
+    getAppMetadata: jest.fn(),
+  },
+}))
 
 describe("contentSecurityPolicy middleware", () => {
   let ctx: any
@@ -57,19 +65,71 @@ describe("contentSecurityPolicy middleware", () => {
   })
 
   it("should handle errors and log an error message", async () => {
+    // Ctx setup to let us try and use CSP whitelist
+    const fakeAppId = "app_sdfdsfsdfsdf"
+    ctx.appId = fakeAppId
+    ctx.user = {
+      license: {
+        features: [Feature.CUSTOM_APP_SCRIPTS],
+      },
+    }
+
     const consoleSpy = jest.spyOn(console, "error").mockImplementation()
     const error = new Error("Test error")
     // @ts-ignore
-    crypto.randomBytes.mockImplementation(() => {
+    app.getAppMetadata.mockImplementation(() => {
       throw error
     })
+    await contentSecurityPolicy(ctx, next)
+
+    expect(app.getAppMetadata).toHaveBeenCalledWith(fakeAppId)
+    expect(consoleSpy).toHaveBeenCalledWith(
+      `Error occurred in Content-Security-Policy middleware: ${error}`
+    )
+    expect(next).toHaveBeenCalled()
+    consoleSpy.mockRestore()
+  })
+
+  it("should add custom CSP whitelist", async () => {
+    const appId = "app_foo"
+    const domain = "https://*.foo.bar"
+
+    // Ctx setup to let us try and use CSP whitelist
+    ctx.appId = appId
+    ctx.user = users.user()
+    ctx.user.license = licenses.license({
+      features: [Feature.CUSTOM_APP_SCRIPTS],
+    })
+
+    // @ts-ignore
+    app.getAppMetadata.mockImplementation(function (): App {
+      return {
+        appId,
+        type: "foo",
+        version: "1",
+        componentLibraries: [],
+        name: "foo",
+        url: "/foo",
+        template: undefined,
+        instance: { _id: appId },
+        tenantId: ctx.user.tenantId,
+        status: "foo",
+        scripts: [
+          {
+            id: "foo",
+            name: "Test",
+            location: "Head",
+            cspWhitelist: domain,
+          },
+        ],
+      }
+    })
 
     await contentSecurityPolicy(ctx, next)
 
-    expect(consoleSpy).toHaveBeenCalledWith(
-      `Error occurred in Content-Security-Policy middleware: ${error}`
-    )
-    expect(next).not.toHaveBeenCalled()
-    consoleSpy.mockRestore()
+    const cspHeader = ctx.set.mock.calls[0][1]
+    expect(cspHeader).toContain(`default-src 'self' ${domain};`)
+    expect(app.getAppMetadata).toHaveBeenCalledWith(appId)
+    expect(next).toHaveBeenCalled()
   })
 })

packages/backend-core/src/users/test/utils.spec.ts

@@ -32,12 +32,12 @@ describe("Users", () => {
     expect(isCreatorSync(user, [])).toBe(true)
   })
 
-  it("User is a creator if it has ADMIN permission in some application", () => {
+  it("User is a not a creator if it has ADMIN permission in some application", () => {
     const user: User = structures.users.user({ roles: { app1: "ADMIN" } })
-    expect(isCreatorSync(user, [])).toBe(true)
+    expect(isCreatorSync(user, [])).toBe(false)
   })
 
-  it("User is a creator if it remains to a group with ADMIN permissions", async () => {
+  it("User is a not a creator if it remains to a group with ADMIN permissions", async () => {
     const usersInGroup = 10
     const groupId = "gr_17abffe89e0b40268e755b952f101a59"
     const group: UserGroup = {
@@ -60,7 +60,7 @@ describe("Users", () => {
       for (let user of users) {
         await db.put(user)
         const creator = (await creatorsInList([user]))[0]
-        expect(creator).toBe(true)
+        expect(creator).toBe(false)
       }
     })
   })

packages/backend-core/src/users/utils.ts

@@ -4,7 +4,6 @@ import env from "../environment"
 import { getExistingAccounts, getFirstPlatformUser } from "./lookup"
 import { EmailUnavailableError } from "../errors"
 import { sdk } from "@budibase/shared-core"
-import { BUILTIN_ROLE_IDS } from "../security/roles"
 import * as context from "../context"
 
 // extract from shared-core to make easily accessible from backend-core
@@ -57,7 +56,7 @@ function isCreatorByGroupMembership(
   )
   if (userGroups && userGroups.length > 0) {
     return userGroups.some(group =>
-      Object.values(group.roles || {}).includes(BUILTIN_ROLE_IDS.ADMIN)
+      Object.values(group.roles || {}).includes("CREATOR")
     )
   }
   return false

packages/builder/src/components/backend/TableNavigator/TableDataImport.svelte

@@ -38,7 +38,7 @@
       },
     },
     [FIELDS.OPTIONS.type]: {
-      label: "Options",
+      label: "Single select",
       value: FIELDS.OPTIONS.type,
       config: {
         type: FIELDS.OPTIONS.type,
@@ -46,7 +46,7 @@
       },
     },
     [FIELDS.ARRAY.type]: {
-      label: "Multi-select",
+      label: "Multi select",
       value: FIELDS.ARRAY.type,
       config: {
         type: FIELDS.ARRAY.type,

packages/builder/src/pages/builder/app/[application]/_components/BuilderSidePanel.svelte

@@ -524,7 +524,7 @@
       return `This user has been given ${role?.name} access from the ${user.group} group`
     }
     if (user.isAdminOrGlobalBuilder) {
-      return "Account admins can edit all apps"
+      return "Workspace admins can edit all apps"
     }
     return null
   }

packages/builder/src/pages/builder/app/[application]/settings/scripts.svelte

@@ -144,6 +144,20 @@
         minHeight={200}
         placeholder="<script>...</script>"
       />
+      <Label size="L">
+        CSP whitelist<br />
+        <Link
+          href="https://docs.budibase.com/docs/app-scripts#domain-whitelisting-for-content-security-policy-csp"
+          target="_blank"
+        >
+          Learn more
+        </Link>
+      </Label>
+      <TextArea
+        bind:value={selectedScript.cspWhitelist}
+        minHeight={100}
+        placeholder="https://external.api.com&#013;https://*.domain.com"
+      />
       <div />
       <div class="buttons">
         {#if !isNew}

packages/builder/src/pages/builder/portal/account/upgrade.svelte

@@ -71,7 +71,11 @@
       notifications.success("Successfully activated")
     } catch (e) {
       console.error(e)
-      notifications.error("Error activating license key")
+      if (e?.status === 409) {
+        notifications.error(e.message)
+      } else {
+        notifications.error("Error activating license key")
+      }
     }
   }
 

packages/frontend-core/src/constants.ts

@@ -52,9 +52,9 @@ export const BudibaseRoleOptionsOld = [
 ]
 export const BudibaseRoleOptions = [
   {
-    label: "Account admin",
+    label: "Workspace admin",
     value: BudibaseRoles.Admin,
-    subtitle: "Has full access to all apps and settings in your account",
+    subtitle: "Has full access to all apps and settings in your workspace",
     sortOrder: 1,
   },
   {

packages/pro

@@ -1 +1 @@
-Subproject commit 1e63fa32ff9635356c1fafd803b6f1e53f282d7d
+Subproject commit 9108e85411fd1b6ba190ff178500637d7e47d759

packages/server/src/api/index.ts

@@ -1,5 +1,10 @@
 import Router from "@koa/router"
-import { auth, middleware, env as envCore } from "@budibase/backend-core"
+import {
+  auth,
+  middleware,
+  env as envCore,
+  env as coreEnv,
+} from "@budibase/backend-core"
 import currentApp from "../middleware/currentapp"
 import cleanup from "../middleware/cleanup"
 import zlib from "zlib"
@@ -66,9 +71,13 @@ if (apiEnabled()) {
     )
     .use(pro.licensing())
     .use(currentApp)
-    .use(auth.auditLog)
-    .use(migrations)
-    .use(cleanup)
+
+  // Add CSP as soon as possible - depends on licensing and currentApp
+  if (!coreEnv.DISABLE_CONTENT_SECURITY_POLICY) {
+    router.use(middleware.csp)
+  }
+
+  router.use(auth.auditLog).use(migrations).use(cleanup)
 
   // authenticated routes
   for (let route of mainRoutes) {

packages/server/src/api/routes/tests/ai.spec.ts

@@ -5,6 +5,7 @@ import { configs, env, features, setEnv } from "@budibase/backend-core"
 import {
   AIInnerConfig,
   ConfigType,
+  Feature,
   License,
   PlanModel,
   PlanType,
@@ -291,7 +292,7 @@ describe("BudibaseAI", () => {
           model: PlanModel.PER_USER,
           usesInvoicing: false,
         },
-        features: [],
+        features: [Feature.BUDIBASE_AI],
         quotas: {} as any,
         tenantId: config.tenantId,
       }

packages/server/src/koa.ts

@@ -6,13 +6,7 @@ import * as api from "./api"
 import * as automations from "./automations"
 import { Thread } from "./threads"
 import * as redis from "./utilities/redis"
-import {
-  events,
-  logging,
-  middleware,
-  timers,
-  env as coreEnv,
-} from "@budibase/backend-core"
+import { events, logging, middleware, timers } from "@budibase/backend-core"
 import destroyable from "server-destroy"
 import { userAgent } from "koa-useragent"
 
@@ -43,9 +37,6 @@ export default function createKoaApp() {
   app.use(middleware.correlation)
   app.use(middleware.pino)
   app.use(middleware.ip)
-  if (!coreEnv.DISABLE_CONTENT_SECURITY_POLICY) {
-    app.use(middleware.csp)
-  }
   app.use(userAgent)
 
   const server = http.createServer(app.callback())

packages/shared-core/src/sdk/documents/users.ts

@@ -68,7 +68,7 @@ export function hasAppCreatorPermissions(user?: User | ContextUser): boolean {
   return _.flow(
     _.get("roles"),
     _.values,
-    _.find(x => ["CREATOR", "ADMIN"].includes(x)),
+    _.find(x => ["CREATOR"].includes(x)),
     x => !!x
   )(user)
 }

packages/types/src/documents/app/app.ts

@@ -110,4 +110,5 @@ export interface AppScript {
   name: string
   location: "Head" | "Body"
   html?: string
+  cspWhitelist?: string
 }