MirrorSave
/
budibase
mirror of https://github.com/Budibase/budibase.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Merge branch 'develop' of github.com:Budibase/budibase into cypress-testing

This commit is contained in:
mike12345567 2022-01-21 13:37:05 +00:00
parent 692a92e052 b10f29e52b
commit 7f1dab078b
50 changed files with 792 additions and 316 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
charts/budibase/templates/app-service-deployment.yaml
View File

@ -99,7 +99,9 @@ spec:
- name: PLATFORM_URL
value: {{ .Values.globals.platformUrl | quote }}
- name: USE_QUOTAS
value: "1"
value: {{ .Values.globals.useQuotas | quote }}
- name: EXCLUDE_QUOTAS_TENANTS
value: {{ .Values.globals.excludeQuotasTenants | quote }}
- name: ACCOUNT_PORTAL_URL
value: {{ .Values.globals.accountPortalUrl | quote }}
- name: ACCOUNT_PORTAL_API_KEY

5
charts/budibase/values.yaml
View File

@ -93,6 +93,8 @@ globals:
logLevel: info
selfHosted: "1" # set to 0 for budibase cloud environment, set to 1 for self-hosted setup
multiTenancy: "0" # set to 0 to disable multiple orgs, set to 1 to enable multiple orgs
useQuotas: "0"
excludeQuotasTenants: "" # comma seperated list of tenants to exclude from quotas
accountPortalUrl: ""
accountPortalApiKey: ""
cookieDomain: ""
@ -239,7 +241,8 @@ couchdb:
hosts:
- chart-example.local
path: /
annotations: []
annotations:
[]
# kubernetes.io/ingress.class: nginx
# kubernetes.io/tls-acme: "true"
tls:

2
lerna.json
View File

@ -1,5 +1,5 @@
{
"version": "1.0.44-alpha.7",
"version": "1.0.46-alpha.3",
"npmClient": "yarn",
"packages": [
"packages/*"

1
packages/backend-core/migrations.js Normal file
View File

@ -0,0 +1 @@
module.exports = require("./src/migrations")

2
packages/backend-core/package.json
View File

@ -1,6 +1,6 @@
{
"name": "@budibase/backend-core",
"version": "1.0.44-alpha.7",
"version": "1.0.46-alpha.3",
"description": "Budibase backend core libraries used in server and worker",
"main": "src/index.js",
"author": "Budibase",

2
packages/backend-core/src/db/constants.js
View File

@ -21,6 +21,7 @@ exports.StaticDatabases = {
name: "global-db",
docs: {
apiKeys: "apikeys",
usageQuota: "usage_quota",
},
},
// contains information about tenancy and so on
@ -28,7 +29,6 @@ exports.StaticDatabases = {
name: "global-info",
docs: {
tenants: "tenants",
usageQuota: "usage_quota",
},
},
}

2
packages/backend-core/src/db/utils.js
View File

@ -450,7 +450,7 @@ async function getScopedConfig(db, params) {
function generateNewUsageQuotaDoc() {
return {
_id: StaticDatabases.PLATFORM_INFO.docs.usageQuota,
_id: StaticDatabases.GLOBAL.docs.usageQuota,
quotaReset: Date.now() + 2592000000,
usageQuota: {
automationRuns: 0,

1
packages/backend-core/src/index.js
View File

@ -14,4 +14,5 @@ module.exports = {
cache: require("../cache"),
auth: require("../auth"),
constants: require("../constants"),
migrations: require("../migrations"),
}

14
packages/backend-core/src/migrations/index.js
View File

@ -1,5 +1,5 @@
const { DocumentTypes } = require("../db/constants")
const { getGlobalDB } = require("../tenancy")
const { getGlobalDB, getTenantId } = require("../tenancy")
exports.MIGRATION_DBS = {
GLOBAL_DB: "GLOBAL_DB",
@ -7,11 +7,13 @@ exports.MIGRATION_DBS = {
exports.MIGRATIONS = {
USER_EMAIL_VIEW_CASING: "user_email_view_casing",
QUOTAS_1: "quotas_1",
}
const DB_LOOKUP = {
[exports.MIGRATION_DBS.GLOBAL_DB]: [
exports.MIGRATIONS.USER_EMAIL_VIEW_CASING,
exports.MIGRATIONS.QUOTAS_1,
],
}
@ -27,6 +29,7 @@ exports.getMigrationsDoc = async db => {
}
exports.migrateIfRequired = async (migrationDb, migrationName, migrateFn) => {
const tenantId = getTenantId()
try {
let db
if (migrationDb === exports.MIGRATION_DBS.GLOBAL_DB) {
@ -47,15 +50,18 @@ exports.migrateIfRequired = async (migrationDb, migrationName, migrateFn) => {
return
}
console.log(`Performing migration: ${migrationName}`)
console.log(`[Tenant: ${tenantId}] Performing migration: ${migrationName}`)
await migrateFn()
console.log(`Migration complete: ${migrationName}`)
console.log(`[Tenant: ${tenantId}] Migration complete: ${migrationName}`)
// mark as complete
doc[migrationName] = Date.now()
await db.put(doc)
} catch (err) {
console.error(`Error performing migration: ${migrationName}: `, err)
console.error(
`[Tenant: ${tenantId}] Error performing migration: ${migrationName}: `,
err
)
throw err
}
}

2
packages/bbui/package.json
View File

@ -1,7 +1,7 @@
{
"name": "@budibase/bbui",
"description": "A UI solution used in the different Budibase projects.",
"version": "1.0.44-alpha.7",
"version": "1.0.46-alpha.3",
"license": "MPL-2.0",
"svelte": "src/index.js",
"module": "dist/bbui.es.js",

8
packages/builder/cypress/support/commands.js
View File

@ -35,7 +35,13 @@ Cypress.Commands.add("login", () => {
Cypress.Commands.add("createApp", name => {
cy.visit(`localhost:${Cypress.env("PORT")}/builder`)
cy.wait(500)
cy.contains(/Start from scratch/).dblclick()
cy.request(`${Cypress.config().baseUrl}api/applications?status=all`)
.its("body")
.then(body => {
if (body.length > 0) {
cy.get(".spectrum-Button").contains("Create app").click({ force: true })
}
})
cy.get(".spectrum-Modal").within(() => {
cy.get("input").eq(0).type(name).should("have.value", name).blur()
cy.get(".spectrum-ButtonGroup").contains("Create app").click()

8
packages/builder/package.json
View File

@ -1,6 +1,6 @@
{
"name": "@budibase/builder",
"version": "1.0.44-alpha.7",
"version": "1.0.46-alpha.3",
"license": "GPL-3.0",
"private": true,
"scripts": {
@ -65,10 +65,10 @@
}
},
"dependencies": {
"@budibase/bbui": "^1.0.44-alpha.7",
"@budibase/client": "^1.0.44-alpha.7",
"@budibase/bbui": "^1.0.46-alpha.3",
"@budibase/client": "^1.0.46-alpha.3",
"@budibase/colorpicker": "1.1.2",
"@budibase/string-templates": "^1.0.44-alpha.7",
"@budibase/string-templates": "^1.0.46-alpha.3",
"@sentry/browser": "5.19.1",
"@spectrum-css/page": "^3.0.1",
"@spectrum-css/vars": "^3.0.1",

1
packages/builder/src/builderStore/api.js
View File

@ -6,6 +6,7 @@ const apiCall =
method =>
async (url, body, headers = { "Content-Type": "application/json" }) => {
headers["x-budibase-app-id"] = svelteGet(store).appId
headers["x-budibase-api-version"] = "1"
const json = headers["Content-Type"] === "application/json"
const resp = await fetch(url, {
method: method,

25
packages/builder/src/components/backend/TableNavigator/modals/CreateTableModal.svelte
View File

@ -53,16 +53,23 @@
}
// Create table
const table = await tables.save(newTable)
notifications.success(`Table ${name} created successfully.`)
analytics.captureEvent(Events.TABLE.CREATED, { name })
let table
try {
table = await tables.save(newTable)
notifications.success(`Table ${name} created successfully.`)
analytics.captureEvent(Events.TABLE.CREATED, { name })
// Navigate to new table
const currentUrl = $url()
const path = currentUrl.endsWith("data")
? `./table/${table._id}`
: `../../table/${table._id}`
$goto(path)
// Navigate to new table
const currentUrl = $url()
const path = currentUrl.endsWith("data")
? `./table/${table._id}`
: `../../table/${table._id}`
$goto(path)
} catch (e) {
notifications.error(e)
// reload in case the table was created
await tables.fetch()
}
}
</script>

4
packages/builder/src/components/design/PropertiesPanel/PropertyControls/ButtonActionEditor/actions/ExecuteQuery.svelte
View File

@ -55,8 +55,8 @@
<div>
<BindingBuilder
bind:customParams={parameters.queryParams}
bindings={query.parameters}
bind:bindableOptions={bindings}
queryBindings={query.parameters}
bind:bindings
/>
<IntegrationQueryEditor
height={200}

4
packages/builder/src/components/design/PropertiesPanel/PropertyControls/DataSourceSelect.svelte
View File

@ -169,8 +169,8 @@
{#if getQueryParams(value).length > 0}
<BindingBuilder
bind:customParams={tmpQueryParams}
bindings={getQueryParams(value)}
bind:bindableOptions={bindings}
queryBindings={getQueryParams(value)}
bind:bindings
/>
{/if}
<IntegrationQueryEditor

21
packages/builder/src/components/integration/QueryBindingBuilder.svelte
View File

@ -7,27 +7,26 @@
import DrawerBindableInput from "components/common/bindings/DrawerBindableInput.svelte"
export let bindable = true
export let queryBindings = []
export let bindings = []
export let bindableOptions = []
export let customParams = {}
function newQueryBinding() {
bindings = [...bindings, {}]
queryBindings = [...queryBindings, {}]
}
$: console.log(bindings)
function deleteQueryBinding(idx) {
bindings.splice(idx, 1)
bindings = bindings
queryBindings.splice(idx, 1)
queryBindings = queryBindings
}
// This is necessary due to the way readable and writable bindings are stored.
// The readable binding in the UI gets converted to a UUID value that the client understands
// for parsing, then converted back so we can display it the readable form in the UI
function onBindingChange(param, valueToParse) {
customParams[param] = readableToRuntimeBinding(
bindableOptions,
valueToParse
)
customParams[param] = readableToRuntimeBinding(bindings, valueToParse)
}
</script>
@ -49,7 +48,7 @@
{/if}
</Body>
<div class="bindings" class:bindable>
{#each bindings as binding, idx}
{#each queryBindings as binding, idx}
<Input
placeholder="Binding Name"
thin
@ -69,10 +68,10 @@
thin
on:change={evt => onBindingChange(binding.name, evt.detail)}
value={runtimeToReadableBinding(
bindableOptions,
bindings,
customParams?.[binding.name]
)}
{bindableOptions}
{bindings}
/>
{:else}
<Icon hoverable name="Close" on:click={() => deleteQueryBinding(idx)} />

2
packages/builder/src/components/integration/QueryViewer.svelte
View File

@ -120,7 +120,7 @@
config={integrationInfo.extra}
/>
{/if}
<BindingBuilder bind:bindings={query.parameters} bindable={false} />
<BindingBuilder bind:queryBindings={query.parameters} bindable={false} />
{/if}
</div>
{#if shouldShowQueryConfig}

83
packages/builder/src/components/start/CreateAppModal.svelte
View File

@ -11,7 +11,6 @@
import { capitalise } from "helpers"
import { goto } from "@roxi/routify"
import { APP_NAME_REGEX } from "constants"
import TemplateList from "./TemplateList.svelte"
export let template
export let inline
@ -147,58 +146,34 @@
}
</script>
{#if showTemplateSelection}
<ModalContent
title={"Get started quickly"}
showConfirmButton={false}
size="L"
onConfirm={() => {
template = {}
return false
}}
showCancelButton={!inline}
showCloseIcon={!inline}
>
<TemplateList
onSelect={(selected, { useImport } = {}) => {
if (!selected) {
template = useImport ? { fromFile: true } : {}
return
}
template = selected
<ModalContent
title={"Name your app"}
confirmText={template?.fromFile ? "Import app" : "Create app"}
onConfirm={createNewApp}
onCancel={inline ? onCancel : null}
cancelText={inline ? "Back" : undefined}
showCloseIcon={!inline}
disabled={!valid}
>
{#if template?.fromFile}
<Dropzone
error={$touched.file && $errors.file}
gallery={false}
label="File to import"
value={[$values.file]}
on:change={e => {
$values.file = e.detail?.[0]
$touched.file = true
}}
/>
</ModalContent>
{:else}
<ModalContent
title={"Name your app"}
confirmText={template?.fromFile ? "Import app" : "Create app"}
onConfirm={createNewApp}
onCancel={inline ? onCancel : null}
cancelText={inline ? "Back" : undefined}
showCloseIcon={!inline}
disabled={!valid}
>
{#if template?.fromFile}
<Dropzone
error={$touched.file && $errors.file}
gallery={false}
label="File to import"
value={[$values.file]}
on:change={e => {
$values.file = e.detail?.[0]
$touched.file = true
}}
/>
{/if}
<Input
bind:value={$values.name}
error={$touched.name && $errors.name}
on:blur={() => ($touched.name = true)}
label="Name"
placeholder={$auth.user.firstName
? `${$auth.user.firstName}'s app`
: "My app"}
/>
</ModalContent>
{/if}
{/if}
<Input
bind:value={$values.name}
error={$touched.name && $errors.name}
on:blur={() => ($touched.name = true)}
label="Name"
placeholder={$auth.user.firstName
? `${$auth.user.firstName}'s app`
: "My app"}
/>
</ModalContent>

69
packages/builder/src/components/start/TemplateList.svelte
View File

@ -1,69 +0,0 @@
<script>
import { Heading, Layout, Icon } from "@budibase/bbui"
export let onSelect
</script>
<Layout gap="XS" noPadding>
<div class="template start-from-scratch" on:click={() => onSelect(null)}>
<div
class="background-icon"
style={`background: rgb(50, 50, 50); color: white;`}
>
<Icon name="Add" />
</div>
<Heading size="XS">Start from scratch</Heading>
<p class="detail">BLANK</p>
</div>
<div
class="template import"
on:click={() => onSelect(null, { useImport: true })}
>
<div
class="background-icon"
style={`background: rgb(50, 50, 50); color: white;`}
>
<Icon name="Add" />
</div>
<Heading size="XS">Import an app</Heading>
<p class="detail">BLANK</p>
</div>
</Layout>
<style>
.background-icon {
padding: 10px;
border-radius: 4px;
display: flex;
align-items: center;
justify-content: center;
width: 18px;
color: white;
}
.template {
min-height: 60px;
display: grid;
grid-gap: var(--layout-s);
grid-template-columns: auto 1fr auto;
border: 1px solid #494949;
align-items: center;
cursor: pointer;
border-radius: 4px;
background: var(--background-alt);
padding: 8px 16px;
}
.detail {
text-align: right;
}
.start-from-scratch {
background: var(--spectrum-global-color-gray-50);
margin-top: 20px;
}
.import {
background: var(--spectrum-global-color-gray-50);
}
</style>

85
packages/builder/src/pages/builder/portal/apps/index.svelte
View File

@ -11,6 +11,7 @@
notifications,
Body,
Search,
Icon,
} from "@budibase/bbui"
import Spinner from "components/common/Spinner.svelte"
import CreateAppModal from "components/start/CreateAppModal.svelte"
@ -27,6 +28,7 @@
import AppRow from "components/start/AppRow.svelte"
import { AppStatus } from "constants"
import analytics, { Events } from "analytics"
import Logo from "assets/bb-space-man.svg"
let sortBy = "name"
let template
@ -78,6 +80,7 @@
}
const initiateAppCreation = () => {
template = {}
creationModal.show()
creatingApp = true
}
@ -300,14 +303,26 @@
<div class="buttons">
{#if cloud}
<Button icon="Export" quiet secondary on:click={initiateAppsExport}>
<Button
size="L"
icon="Export"
quiet
secondary
on:click={initiateAppsExport}
>
Export apps
</Button>
{/if}
<Button icon="Import" quiet secondary on:click={initiateAppImport}>
<Button
icon="Import"
size="L"
quiet
secondary
on:click={initiateAppImport}
>
Import app
</Button>
<Button icon="Add" cta on:click={initiateAppCreation}>
<Button size="L" icon="Add" cta on:click={initiateAppCreation}>
Create app
</Button>
</div>
@ -389,9 +404,24 @@
{#if !enrichedApps.length && !creatingApp && loaded}
<div class="empty-wrapper">
<Modal inline>
<CreateAppModal {template} inline={true} />
</Modal>
<div class="centered">
<div class="main">
<Layout gap="S" justifyItems="center">
<img class="img-size" alt="logo" src={Logo} />
<div class="new-screen-text">
<Detail size="M">Create a business app in minutes!</Detail>
</div>
<Button on:click={() => initiateAppCreation()} size="M" cta>
<div class="new-screen-button">
<div class="background-icon" style="color: white;">
<Icon name="Add" />
</div>
Create App
</div></Button
>
</Layout>
</div>
</div>
</div>
{/if}
@ -474,12 +504,15 @@
}
.grid {
height: 200px;
display: grid;
overflow: hidden;
grid-gap: var(--spacing-xl);
grid-template-columns: repeat(auto-fill, minmax(270px, 1fr));
grid-template-rows: minmax(70px, 1fr) minmax(100px, 1fr) minmax(0px, 0);
}
.template-card {
height: 80px;
height: 70px;
border-radius: var(--border-radius-s);
border: 1px solid var(--spectrum-global-color-gray-300);
cursor: pointer;
@ -533,4 +566,42 @@
justify-content: center;
align-items: center;
}
.centered {
width: calc(100% - 350px);
height: calc(100% - 100px);
display: flex;
flex-direction: column;
justify-content: center;
align-items: center;
}
.main {
width: 300px;
}
.new-screen-text {
width: 160px;
text-align: center;
color: #2c2c2c;
font-weight: 600;
}
.new-screen-button {
margin-left: 5px;
height: 20px;
width: 100px;
display: flex;
align-items: center;
}
.img-size {
width: 160px;
height: 160px;
}
.background-icon {
margin-top: 4px;
margin-right: 4px;
}
</style>

2
packages/cli/package.json
View File

@ -1,6 +1,6 @@
{
"name": "@budibase/cli",
"version": "1.0.44-alpha.7",
"version": "1.0.46-alpha.3",
"description": "Budibase CLI, for developers, self hosting and migrations.",
"main": "src/index.js",
"bin": {

6
packages/client/package.json
View File

@ -1,6 +1,6 @@
{
"name": "@budibase/client",
"version": "1.0.44-alpha.7",
"version": "1.0.46-alpha.3",
"license": "MPL-2.0",
"module": "dist/budibase-client.js",
"main": "dist/budibase-client.js",
@ -19,9 +19,9 @@
"dev:builder": "rollup -cw"
},
"dependencies": {
"@budibase/bbui": "^1.0.44-alpha.7",
"@budibase/bbui": "^1.0.46-alpha.3",
"@budibase/standard-components": "^0.9.139",
"@budibase/string-templates": "^1.0.44-alpha.7",
"@budibase/string-templates": "^1.0.46-alpha.3",
"regexparam": "^1.3.0",
"shortid": "^2.2.15",
"svelte-spa-router": "^3.0.5"

6
packages/client/src/utils/fetch/DataFetch.js
View File

@ -110,12 +110,6 @@ export default class DataFetch {
*/
async getInitialData() {
const { datasource, filter, sortColumn, paginate } = this.options
const tableId = datasource?.tableId
// Ensure table ID exists
if (!tableId) {
return
}
// Fetch datasource definition and determine feature flags
const definition = await this.constructor.getDefinition(datasource)

8
packages/server/package.json
View File

@ -1,7 +1,7 @@
{
"name": "@budibase/server",
"email": "hi@budibase.com",
"version": "1.0.44-alpha.7",
"version": "1.0.46-alpha.3",
"description": "Budibase Web Server",
"main": "src/index.ts",
"repository": {
@ -70,9 +70,9 @@
"license": "GPL-3.0",
"dependencies": {
"@apidevtools/swagger-parser": "^10.0.3",
"@budibase/backend-core": "^1.0.44-alpha.7",
"@budibase/client": "^1.0.44-alpha.7",
"@budibase/string-templates": "^1.0.44-alpha.7",
"@budibase/backend-core": "^1.0.46-alpha.3",
"@budibase/client": "^1.0.46-alpha.3",
"@budibase/string-templates": "^1.0.46-alpha.3",
"@bull-board/api": "^3.7.0",
"@bull-board/koa": "^3.7.0",
"@elastic/elasticsearch": "7.10.0",

2
packages/server/src/api/controllers/table/internal.js
View File

@ -8,6 +8,7 @@ const {
getTable,
handleDataImport,
} = require("./utils")
const usageQuota = require("../../../utilities/usageQuota")
exports.save = async function (ctx) {
const appId = ctx.appId
@ -119,6 +120,7 @@ exports.destroy = async function (ctx) {
})
)
await db.bulkDocs(rows.rows.map(row => ({ ...row.doc, _deleted: true })))
await usageQuota.update(usageQuota.Properties.ROW, -rows.rows.length)
// update linked rows
await linkRows.updateLinks({

5
packages/server/src/api/controllers/table/utils.js
View File

@ -16,6 +16,7 @@ const {
} = require("../../../integrations/utils")
const { getViews, saveView } = require("../view/utils")
const viewTemplate = require("../view/viewBuilder")
const usageQuota = require("../../../utilities/usageQuota")
exports.checkForColumnUpdates = async (db, oldTable, updatedTable) => {
let updatedRows = []
@ -112,7 +113,11 @@ exports.handleDataImport = async (appId, user, table, dataImport) => {
finalData.push(row)
}
await usageQuota.update(usageQuota.Properties.ROW, finalData.length, {
dryRun: true,
})
await db.bulkDocs(finalData)
await usageQuota.update(usageQuota.Properties.ROW, finalData.length)
let response = await db.put(table)
table._rev = response._rev
return table

6
packages/server/src/automations/steps/createRow.js
View File

@ -1,6 +1,5 @@
const rowController = require("../../api/controllers/row")
const automationUtils = require("../automationUtils")
const env = require("../../environment")
const usage = require("../../utilities/usageQuota")
const { buildCtx } = require("./utils")
@ -83,10 +82,9 @@ exports.run = async function ({ inputs, appId, emitter }) {
inputs.row.tableId,
inputs.row
)
if (env.USE_QUOTAS) {
await usage.update(usage.Properties.ROW, 1)
}
await usage.update(usage.Properties.ROW, 1, { dryRun: true })
await rowController.save(ctx)
await usage.update(usage.Properties.ROW, 1)
return {
row: inputs.row,
response: ctx.body,

5
packages/server/src/automations/steps/deleteRow.js
View File

@ -1,5 +1,4 @@
const rowController = require("../../api/controllers/row")
const env = require("../../environment")
const usage = require("../../utilities/usageQuota")
const { buildCtx } = require("./utils")
const automationUtils = require("../automationUtils")
@ -74,9 +73,7 @@ exports.run = async function ({ inputs, appId, emitter }) {
})
try {
if (env.isProd()) {
await usage.update(usage.Properties.ROW, -1)
}
await usage.update(usage.Properties.ROW, -1)
await rowController.destroy(ctx)
return {
response: ctx.body,

1
packages/server/src/environment.js
View File

@ -38,6 +38,7 @@ module.exports = {
MINIO_ACCESS_KEY: process.env.MINIO_ACCESS_KEY,
MINIO_SECRET_KEY: process.env.MINIO_SECRET_KEY,
USE_QUOTAS: process.env.USE_QUOTAS,
EXCLUDE_QUOTAS_TENANTS: process.env.EXCLUDE_QUOTAS_TENANTS,
REDIS_URL: process.env.REDIS_URL,
REDIS_PASSWORD: process.env.REDIS_PASSWORD,
INTERNAL_API_KEY: process.env.INTERNAL_API_KEY,

40
packages/server/src/middleware/tests/usageQuota.spec.js
View File

@ -1,11 +1,5 @@
jest.mock("../../db")
jest.mock("../../utilities/usageQuota")
jest.mock("../../environment", () => ({
isTest: () => true,
isProd: () => false,
isDev: () => true,
_set: () => {},
}))
jest.mock("@budibase/backend-core/tenancy", () => ({
getTenantId: () => "testing123"
}))
@ -29,9 +23,10 @@ class TestConfiguration {
},
req: {
method: "POST",
url: "/rows"
url: "/applications"
}
}
usageQuota.useQuotas = () => true
}
executeMiddleware() {
@ -113,7 +108,6 @@ describe("usageQuota middleware", () => {
it("calculates and persists the correct usage quota for the relevant action", async () => {
config.setUrl("/rows")
config.setProd(true)
await config.executeMiddleware()
@ -121,20 +115,20 @@ describe("usageQuota middleware", () => {
expect(config.next).toHaveBeenCalled()
})
it("calculates the correct file size from a file upload call and adds it to quota", async () => {
config.setUrl("/upload")
config.setProd(true)
config.setFiles([
{
size: 100
},
{
size: 10000
},
])
await config.executeMiddleware()
// it("calculates the correct file size from a file upload call and adds it to quota", async () => {
// config.setUrl("/upload")
// config.setProd(true)
// config.setFiles([
// {
// size: 100
// },
// {
// size: 10000
// },
// ])
// await config.executeMiddleware()
expect(usageQuota.update).toHaveBeenCalledWith("storage", 10100)
expect(config.next).toHaveBeenCalled()
})
// expect(usageQuota.update).toHaveBeenCalledWith("storage", 10100)
// expect(config.next).toHaveBeenCalled()
// })
})

108
packages/server/src/middleware/usageQuota.js
View File

@ -1,14 +1,11 @@
const CouchDB = require("../db")
const usageQuota = require("../utilities/usageQuota")
const env = require("../environment")
const { getTenantId } = require("@budibase/backend-core/tenancy")
const { getUniqueRows } = require("../utilities/usageQuota/rows")
const {
isExternalTable,
isRowId: isExternalRowId,
} = require("../integrations/utils")
// tenants without limits
const EXCLUDED_TENANTS = ["bb", "default", "bbtest", "bbstaging"]
const migration = require("../migrations/usageQuotas")
// currently only counting new writes and deletes
const METHOD_MAP = {
@ -18,13 +15,13 @@ const METHOD_MAP = {
const DOMAIN_MAP = {
rows: usageQuota.Properties.ROW,
upload: usageQuota.Properties.UPLOAD,
views: usageQuota.Properties.VIEW,
users: usageQuota.Properties.USER,
// upload: usageQuota.Properties.UPLOAD, // doesn't work yet
// views: usageQuota.Properties.VIEW, // doesn't work yet
// users: usageQuota.Properties.USER, // doesn't work yet
applications: usageQuota.Properties.APPS,
// this will not be updated by endpoint calls
// instead it will be updated by triggerInfo
automationRuns: usageQuota.Properties.AUTOMATION,
// automationRuns: usageQuota.Properties.AUTOMATION, // doesn't work yet
}
function getProperty(url) {
@ -36,10 +33,7 @@ function getProperty(url) {
}
module.exports = async (ctx, next) => {
const tenantId = getTenantId()
// if in development or a self hosted cloud usage quotas should not be executed
if (env.isDev() || env.SELF_HOSTED || EXCLUDED_TENANTS.includes(tenantId)) {
if (!usageQuota.useQuotas()) {
return next()
}
@ -80,9 +74,93 @@ module.exports = async (ctx, next) => {
usage = files.map(file => file.size).reduce((total, size) => total + size)
}
try {
await usageQuota.update(property, usage)
return next()
await migration.run()
await performRequest(ctx, next, property, usage)
} catch (err) {
ctx.throw(400, err)
}
}
const performRequest = async (ctx, next, property, usage) => {
const usageContext = {
skipNext: false,
skipUsage: false,
[usageQuota.Properties.APPS]: {},
}
if (usage === -1) {
if (PRE_DELETE[property]) {
await PRE_DELETE[property](ctx, usageContext)
}
} else {
if (PRE_CREATE[property]) {
await PRE_CREATE[property](ctx, usageContext)
}
}
// run the request
if (!usageContext.skipNext) {
await usageQuota.update(property, usage, { dryRun: true })
await next()
}
if (usage === -1) {
if (POST_DELETE[property]) {
await POST_DELETE[property](ctx, usageContext)
}
} else {
if (POST_CREATE[property]) {
await POST_CREATE[property](ctx, usageContext)
}
}
// update the usage
if (!usageContext.skipUsage) {
await usageQuota.update(property, usage)
}
}
const appPreDelete = async (ctx, usageContext) => {
if (ctx.query.unpublish) {
// don't run usage decrement for unpublish
usageContext.skipUsage = true
return
}
// store the row count to delete
const rows = await getUniqueRows([ctx.appId])
if (rows.length) {
usageContext[usageQuota.Properties.APPS] = { rowCount: rows.length }
}
}
const appPostDelete = async (ctx, usageContext) => {
// delete the app rows from usage
const rowCount = usageContext[usageQuota.Properties.APPS].rowCount
if (rowCount) {
await usageQuota.update(usageQuota.Properties.ROW, -rowCount)
}
}
const appPostCreate = async ctx => {
// app import & template creation
if (ctx.request.body.useTemplate === "true") {
const rows = await getUniqueRows([ctx.response.body.appId])
const rowCount = rows ? rows.length : 0
await usageQuota.update(usageQuota.Properties.ROW, rowCount)
}
}
const PRE_DELETE = {
[usageQuota.Properties.APPS]: appPreDelete,
}
const POST_DELETE = {
[usageQuota.Properties.APPS]: appPostDelete,
}
const PRE_CREATE = {}
const POST_CREATE = {
[usageQuota.Properties.APPS]: appPostCreate,
}

27
packages/server/src/migrations/tests/usageQuotas/index.spec.js Normal file
View File

@ -0,0 +1,27 @@
const env = require("../../../environment")
const TestConfig = require("../../../tests/utilities/TestConfiguration")
const syncApps = jest.fn()
const syncRows = jest.fn()
jest.mock("../../usageQuotas/syncApps", () => ({ run: syncApps }) )
jest.mock("../../usageQuotas/syncRows", () => ({ run: syncRows }) )
const migrations = require("../../usageQuotas")
describe("run", () => {
let config = new TestConfig(false)
beforeEach(async () => {
await config.init()
env._set("USE_QUOTAS", 1)
})
afterAll(config.end)
it("runs the required migrations", async () => {
await migrations.run()
expect(syncApps).toHaveBeenCalledTimes(1)
expect(syncRows).toHaveBeenCalledTimes(1)
})
})

37
packages/server/src/migrations/tests/usageQuotas/syncApps.spec.js Normal file
View File

@ -0,0 +1,37 @@
const { getGlobalDB } = require("@budibase/backend-core/tenancy")
const TestConfig = require("../../../tests/utilities/TestConfiguration")
const { getUsageQuotaDoc, update, Properties } = require("../../../utilities/usageQuota")
const syncApps = require("../../usageQuotas/syncApps")
const env = require("../../../environment")
describe("syncApps", () => {
let config = new TestConfig(false)
beforeEach(async () => {
await config.init()
env._set("USE_QUOTAS", 1)
})
afterAll(config.end)
it("runs successfully", async () => {
// create the usage quota doc and mock usages
const db = getGlobalDB()
await getUsageQuotaDoc(db)
await update(Properties.APPS, 3)
let usageDoc = await getUsageQuotaDoc(db)
expect(usageDoc.usageQuota.apps).toEqual(3)
// create an extra app to test the migration
await config.createApp("quota-test")
// migrate
await syncApps.run()
// assert the migration worked
usageDoc = await getUsageQuotaDoc(db)
expect(usageDoc.usageQuota.apps).toEqual(2)
})
})

43
packages/server/src/migrations/tests/usageQuotas/syncRows.spec.js Normal file
View File

@ -0,0 +1,43 @@
const { getGlobalDB } = require("@budibase/backend-core/tenancy")
const TestConfig = require("../../../tests/utilities/TestConfiguration")
const { getUsageQuotaDoc, update, Properties } = require("../../../utilities/usageQuota")
const syncRows = require("../../usageQuotas/syncRows")
const env = require("../../../environment")
describe("syncRows", () => {
let config = new TestConfig(false)
beforeEach(async () => {
await config.init()
env._set("USE_QUOTAS", 1)
})
afterAll(config.end)
it("runs successfully", async () => {
// create the usage quota doc and mock usages
const db = getGlobalDB()
await getUsageQuotaDoc(db)
await update(Properties.ROW, 300)
let usageDoc = await getUsageQuotaDoc(db)
expect(usageDoc.usageQuota.rows).toEqual(300)
// app 1
await config.createTable()
await config.createRow()
// app 2
await config.createApp()
await config.createTable()
await config.createRow()
await config.createRow()
// migrate
await syncRows.run()
// assert the migration worked
usageDoc = await getUsageQuotaDoc(db)
expect(usageDoc.usageQuota.rows).toEqual(3)
})
})

24
packages/server/src/migrations/usageQuotas/index.js Normal file
View File

@ -0,0 +1,24 @@
const {
MIGRATIONS,
MIGRATION_DBS,
migrateIfRequired,
} = require("@budibase/backend-core/migrations")
const { useQuotas } = require("../../utilities/usageQuota")
const syncApps = require("./syncApps")
const syncRows = require("./syncRows")
exports.run = async () => {
if (!useQuotas()) {
return
}
// Jan 2022
await migrateIfRequired(
MIGRATION_DBS.GLOBAL_DB,
MIGRATIONS.QUOTAS_1,
async () => {
await syncApps.run()
await syncRows.run()
}
)
}

18
packages/server/src/migrations/usageQuotas/syncApps.js Normal file
View File

@ -0,0 +1,18 @@
const { getGlobalDB, getTenantId } = require("@budibase/backend-core/tenancy")
const { getAllApps } = require("@budibase/backend-core/db")
const CouchDB = require("../../db")
const { getUsageQuotaDoc } = require("../../utilities/usageQuota")
exports.run = async () => {
const db = getGlobalDB()
// get app count
const devApps = await getAllApps(CouchDB, { dev: true })
const appCount = devApps ? devApps.length : 0
// sync app count
const tenantId = getTenantId()
console.log(`[Tenant: ${tenantId}] Syncing app count: ${appCount}`)
const usageDoc = await getUsageQuotaDoc(db)
usageDoc.usageQuota.apps = appCount
await db.put(usageDoc)
}

21
packages/server/src/migrations/usageQuotas/syncRows.js Normal file
View File

@ -0,0 +1,21 @@
const { getGlobalDB, getTenantId } = require("@budibase/backend-core/tenancy")
const { getAllApps } = require("@budibase/backend-core/db")
const CouchDB = require("../../db")
const { getUsageQuotaDoc } = require("../../utilities/usageQuota")
const { getUniqueRows } = require("../../utilities/usageQuota/rows")
exports.run = async () => {
const db = getGlobalDB()
// get all rows in all apps
const allApps = await getAllApps(CouchDB, { all: true })
const appIds = allApps ? allApps.map(app => app.appId) : []
const rows = await getUniqueRows(appIds)
const rowCount = rows ? rows.length : 0
// sync row count
const tenantId = getTenantId()
console.log(`[Tenant: ${tenantId}] Syncing row count: ${rowCount}`)
const usageDoc = await getUsageQuotaDoc(db)
usageDoc.usageQuota.rows = rowCount
await db.put(usageDoc)
}

72
packages/server/src/utilities/tests/usageQuota/usageQuota.spec.js Normal file
View File

@ -0,0 +1,72 @@
const getTenantId = jest.fn()
jest.mock("@budibase/backend-core/tenancy", () => ({
getTenantId
}))
const usageQuota = require("../../usageQuota")
const env = require("../../../environment")
class TestConfiguration {
constructor() {
this.enableQuotas()
}
enableQuotas = () => {
env.USE_QUOTAS = 1
}
disableQuotas = () => {
env.USE_QUOTAS = null
}
setTenantId = (tenantId) => {
getTenantId.mockReturnValue(tenantId)
}
setExcludedTenants = (tenants) => {
env.EXCLUDE_QUOTAS_TENANTS = tenants
}
reset = () => {
this.disableQuotas()
this.setExcludedTenants(null)
}
}
describe("usageQuota", () => {
let config
beforeEach(() => {
config = new TestConfiguration()
})
afterEach(() => {
config.reset()
})
describe("useQuotas", () => {
it("works when no settings have been provided", () => {
config.reset()
expect(usageQuota.useQuotas()).toBe(false)
})
it("honours USE_QUOTAS setting", () => {
config.disableQuotas()
expect(usageQuota.useQuotas()).toBe(false)
config.enableQuotas()
expect(usageQuota.useQuotas()).toBe(true)
})
it("honours EXCLUDE_QUOTAS_TENANTS setting", () => {
config.setTenantId("test")
// tenantId is in the list
config.setExcludedTenants("test, test2, test2")
expect(usageQuota.useQuotas()).toBe(false)
config.setExcludedTenants("test,test2,test2")
expect(usageQuota.useQuotas()).toBe(false)
// tenantId is not in the list
config.setTenantId("other")
expect(usageQuota.useQuotas()).toBe(true)
})
})
})

73
packages/server/src/utilities/usageQuota.js
View File

@ -1,73 +0,0 @@
const env = require("../environment")
const { getGlobalDB } = require("@budibase/backend-core/tenancy")
const {
StaticDatabases,
generateNewUsageQuotaDoc,
} = require("@budibase/backend-core/db")
function getNewQuotaReset() {
return Date.now() + 2592000000
}
exports.Properties = {
ROW: "rows",
UPLOAD: "storage",
VIEW: "views",
USER: "users",
AUTOMATION: "automationRuns",
APPS: "apps",
EMAILS: "emails",
}
async function getUsageQuotaDoc(db) {
let quota
try {
quota = await db.get(StaticDatabases.PLATFORM_INFO.docs.usageQuota)
} catch (err) {
// doc doesn't exist. Create it
quota = await db.post(generateNewUsageQuotaDoc())
}
return quota
}
/**
* Given a specified tenantId this will add to the usage object for the specified property.
* @param {string} property The property which is to be added to (within the nested usageQuota object).
* @param {number} usage The amount (this can be negative) to adjust the number by.
* @returns {Promise<void>} When this completes the API key will now be up to date - the quota period may have
* also been reset after this call.
*/
exports.update = async (property, usage) => {
if (!env.USE_QUOTAS) {
return
}
try {
const db = getGlobalDB()
const quota = await getUsageQuotaDoc(db)
// Check if the quota needs reset
if (Date.now() >= quota.quotaReset) {
quota.quotaReset = getNewQuotaReset()
for (let prop of Object.keys(quota.usageQuota)) {
quota.usageQuota[prop] = 0
}
}
// increment the quota
quota.usageQuota[property] += usage
if (quota.usageQuota[property] > quota.usageLimits[property]) {
throw new Error(
`You have exceeded your usage quota of ${quota.usageLimits[property]} ${property}.`
)
}
// update the usage quotas
await db.put(quota)
} catch (err) {
console.error(`Error updating usage quotas for ${property}`, err)
throw err
}
}

92
packages/server/src/utilities/usageQuota/index.js Normal file
View File

@ -0,0 +1,92 @@
const env = require("../../environment")
const { getGlobalDB, getTenantId } = require("@budibase/backend-core/tenancy")
const {
StaticDatabases,
generateNewUsageQuotaDoc,
} = require("@budibase/backend-core/db")
exports.useQuotas = () => {
// check if quotas are enabled
if (env.USE_QUOTAS) {
// check if there are any tenants without limits
if (env.EXCLUDE_QUOTAS_TENANTS) {
const excludedTenants = env.EXCLUDE_QUOTAS_TENANTS.replace(
/\s/g,
""
).split(",")
const tenantId = getTenantId()
if (excludedTenants.includes(tenantId)) {
return false
}
}
return true
}
return false
}
exports.Properties = {
ROW: "rows",
UPLOAD: "storage", // doesn't work yet
VIEW: "views", // doesn't work yet
USER: "users", // doesn't work yet
AUTOMATION: "automationRuns", // doesn't work yet
APPS: "apps",
EMAILS: "emails", // doesn't work yet
}
exports.getUsageQuotaDoc = async db => {
let quota
try {
quota = await db.get(StaticDatabases.GLOBAL.docs.usageQuota)
} catch (err) {
// doc doesn't exist. Create it
quota = generateNewUsageQuotaDoc()
const response = await db.put(quota)
quota._rev = response.rev
}
return quota
}
/**
* Given a specified tenantId this will add to the usage object for the specified property.
* @param {string} property The property which is to be added to (within the nested usageQuota object).
* @param {number} usage The amount (this can be negative) to adjust the number by.
* @returns {Promise<void>} When this completes the API key will now be up to date - the quota period may have
* also been reset after this call.
*/
exports.update = async (property, usage, opts = { dryRun: false }) => {
if (!exports.useQuotas()) {
return
}
try {
const db = getGlobalDB()
const quota = await exports.getUsageQuotaDoc(db)
// increment the quota
quota.usageQuota[property] += usage
if (
quota.usageQuota[property] > quota.usageLimits[property] &&
usage > 0 // allow for decrementing usage when the quota is already exceeded
) {
throw new Error(
`You have exceeded your usage quota of ${quota.usageLimits[property]} ${property}.`
)
}
if (quota.usageQuota[property] < 0) {
// never go negative if the quota has previously been exceeded
quota.usageQuota[property] = 0
}
// update the usage quotas
if (!opts.dryRun) {
await db.put(quota)
}
} catch (err) {
console.error(`Error updating usage quotas for ${property}`, err)
throw err
}
}

74
packages/server/src/utilities/usageQuota/rows.js Normal file
View File

@ -0,0 +1,74 @@
const { getRowParams, USER_METDATA_PREFIX } = require("../../db/utils")
const CouchDB = require("../../db")
const { isDevAppID, getDevelopmentAppID } = require("@budibase/backend-core/db")
const ROW_EXCLUSIONS = [USER_METDATA_PREFIX]
const getAppPairs = appIds => {
// collect the app ids into dev / prod pairs
// keyed by the dev app id
const pairs = {}
for (let appId of appIds) {
const devId = getDevelopmentAppID(appId)
if (!pairs[devId]) {
pairs[devId] = {}
}
if (isDevAppID(appId)) {
pairs[devId].devId = appId
} else {
pairs[devId].prodId = appId
}
}
return pairs
}
const getAppRows = async appId => {
const appDb = new CouchDB(appId)
const response = await appDb.allDocs(
getRowParams(null, null, {
include_docs: false,
})
)
return response.rows
.map(r => r.id)
.filter(id => {
for (let exclusion of ROW_EXCLUSIONS) {
if (id.startsWith(exclusion)) {
return false
}
}
return true
})
}
/**
* Return a set of all rows in the given app ids.
* The returned rows will be unique on a per dev/prod app basis.
* Rows duplicates may exist across apps due to data import so they are not filtered out.
*/
exports.getUniqueRows = async appIds => {
let uniqueRows = []
const pairs = getAppPairs(appIds)
for (let pair of Object.values(pairs)) {
let appRows = []
for (let appId of [pair.devId, pair.prodId]) {
if (!appId) {
continue
}
try {
appRows.push(await getAppRows(appId))
} catch (e) {
console.error(e)
// don't error out if we can't count the app rows, just continue
}
}
// ensure uniqueness on a per app pair basis
// this can't be done on all rows because app import results in
// duplicate row ids across apps
uniqueRows = uniqueRows.concat(...new Set(appRows))
}
return uniqueRows
}

18
packages/server/src/utilities/usageQuota/usageQuoteReset.js Normal file
View File

@ -0,0 +1,18 @@
// UNUSED CODE
// Preserved for future use
/* eslint-disable no-unused-vars */
function getNewQuotaReset() {
return Date.now() + 2592000000
}
function resetQuotasIfRequired(quota) {
// Check if the quota needs reset
if (Date.now() >= quota.quotaReset) {
quota.quotaReset = getNewQuotaReset()
for (let prop of Object.keys(quota.usageQuota)) {
quota.usageQuota[prop] = 0
}
}
}

2
packages/server/src/utilities/workerRequests.js
View File

@ -71,7 +71,7 @@ exports.getDeployedApps = async () => {
for (let [key, value] of Object.entries(json)) {
if (value.url) {
value.url = value.url.toLowerCase()
apps[key] = value
apps[key.toLowerCase()] = value
}
}
return apps

2
packages/string-templates/package.json
View File

@ -1,6 +1,6 @@
{
"name": "@budibase/string-templates",
"version": "1.0.44-alpha.7",
"version": "1.0.46-alpha.3",
"description": "Handlebars wrapper for Budibase templating.",
"main": "src/index.cjs",
"module": "dist/bundle.mjs",

6
packages/string-templates/src/helpers/javascript.js
View File

@ -1,5 +1,6 @@
const { atob } = require("../utilities")
const { cloneDeep } = require("lodash/fp")
const { LITERAL_MARKER } = require("../helpers/constants")
// The method of executing JS scripts depends on the bundle being built.
// This setter is used in the entrypoint (either index.cjs or index.mjs).
@ -46,8 +47,9 @@ module.exports.processJS = (handlebars, context) => {
$: path => getContextValue(path, cloneDeep(context)),
}
// Create a sandbox with out context and run the JS
return runJS(js, sandboxContext)
// Create a sandbox with our context and run the JS
const res = { data: runJS(js, sandboxContext) }
return `{{${LITERAL_MARKER} js_result-${JSON.stringify(res)}}}`
} catch (error) {
return "Error while executing JS"
}

5
packages/string-templates/src/processors/postprocessor.js
View File

@ -36,6 +36,11 @@ module.exports.processors = [
return value === "true"
case "object":
return JSON.parse(value)
case "js_result":
// We use the literal helper to process the result of JS expressions
// as we want to be able to return any types.
// We wrap the value in an abject to be able to use undefined properly.
return JSON.parse(value).data
}
return value
}),

48
packages/string-templates/test/javascript.spec.js
View File

@ -7,7 +7,7 @@ const processJS = (js, context) => {
describe("Test the JavaScript helper", () => {
it("should execute a simple expression", () => {
const output = processJS(`return 1 + 2`)
expect(output).toBe("3")
expect(output).toBe(3)
})
it("should be able to use primitive bindings", () => {
@ -50,6 +50,52 @@ describe("Test the JavaScript helper", () => {
expect(output).toBe("shazbat")
})
it("should be able to return an object", () => {
const output = processJS(`return $("foo")`, {
foo: {
bar: {
baz: "shazbat",
},
},
})
expect(output.bar.baz).toBe("shazbat")
})
it("should be able to return an array", () => {
const output = processJS(`return $("foo")`, {
foo: ["a", "b", "c"],
})
expect(output[2]).toBe("c")
})
it("should be able to return null", () => {
const output = processJS(`return $("foo")`, {
foo: null,
})
expect(output).toBe(null)
})
it("should be able to return undefined", () => {
const output = processJS(`return $("foo")`, {
foo: undefined,
})
expect(output).toBe(undefined)
})
it("should be able to return 0", () => {
const output = processJS(`return $("foo")`, {
foo: 0,
})
expect(output).toBe(0)
})
it("should be able to return an empty string", () => {
const output = processJS(`return $("foo")`, {
foo: "",
})
expect(output).toBe("")
})
it("should be able to use a deep array binding", () => {
const output = processJS(`return $("foo.0.bar")`, {
foo: [

6
packages/worker/package.json
View File

@ -1,7 +1,7 @@
{
"name": "@budibase/worker",
"email": "hi@budibase.com",
"version": "1.0.44-alpha.7",
"version": "1.0.46-alpha.3",
"description": "Budibase background service",
"main": "src/index.js",
"repository": {
@ -29,8 +29,8 @@
"author": "Budibase",
"license": "GPL-3.0",
"dependencies": {
"@budibase/backend-core": "^1.0.44-alpha.7",
"@budibase/string-templates": "^1.0.44-alpha.7",
"@budibase/backend-core": "^1.0.46-alpha.3",
"@budibase/string-templates": "^1.0.46-alpha.3",
"@koa/router": "^8.0.0",
"@sentry/node": "^6.0.0",
"@techpass/passport-openidconnect": "^0.3.0",

6
packages/worker/src/api/controllers/global/users.js
View File

@ -73,16 +73,14 @@ exports.adminUser = async ctx => {
if (!env.SELF_HOSTED) {
// could be a scenario where it exists, make sure its clean
try {
const usageQuota = await db.get(
StaticDatabases.PLATFORM_INFO.docs.usageQuota
)
const usageQuota = await db.get(StaticDatabases.GLOBAL.docs.usageQuota)
if (usageQuota) {
await db.remove(usageQuota._id, usageQuota._rev)
}
} catch (err) {
// don't worry about errors
}
await db.post(generateNewUsageQuotaDoc())
await db.put(generateNewUsageQuotaDoc())
}
if (response.rows.some(row => row.doc.admin)) {