Merge branch 'develop' into feature/app-urls

This commit is contained in:
Rory Powell 2022-01-21 17:35:23 +00:00
commit 81e8ceff1a
45 changed files with 677 additions and 184 deletions

View File

@ -99,7 +99,9 @@ spec:
- name: PLATFORM_URL
value: {{ .Values.globals.platformUrl | quote }}
- name: USE_QUOTAS
value: "1"
value: {{ .Values.globals.useQuotas | quote }}
- name: EXCLUDE_QUOTAS_TENANTS
value: {{ .Values.globals.excludeQuotasTenants | quote }}
- name: ACCOUNT_PORTAL_URL
value: {{ .Values.globals.accountPortalUrl | quote }}
- name: ACCOUNT_PORTAL_API_KEY

View File

@ -93,6 +93,8 @@ globals:
logLevel: info
selfHosted: "1" # set to 0 for budibase cloud environment, set to 1 for self-hosted setup
multiTenancy: "0" # set to 0 to disable multiple orgs, set to 1 to enable multiple orgs
useQuotas: "0"
excludeQuotasTenants: "" # comma seperated list of tenants to exclude from quotas
accountPortalUrl: ""
accountPortalApiKey: ""
cookieDomain: ""
@ -239,7 +241,8 @@ couchdb:
hosts:
- chart-example.local
path: /
annotations: []
annotations:
[]
# kubernetes.io/ingress.class: nginx
# kubernetes.io/tls-acme: "true"
tls:

View File

@ -1,5 +1,5 @@
{
"version": "1.0.44-alpha.9",
"version": "1.0.46-alpha.3",
"npmClient": "yarn",
"packages": [
"packages/*"

View File

@ -0,0 +1 @@
module.exports = require("./src/migrations")

View File

@ -1,6 +1,6 @@
{
"name": "@budibase/backend-core",
"version": "1.0.44-alpha.9",
"version": "1.0.46-alpha.3",
"description": "Budibase backend core libraries used in server and worker",
"main": "src/index.js",
"author": "Budibase",

View File

@ -21,6 +21,7 @@ exports.StaticDatabases = {
name: "global-db",
docs: {
apiKeys: "apikeys",
usageQuota: "usage_quota",
},
},
// contains information about tenancy and so on
@ -28,7 +29,6 @@ exports.StaticDatabases = {
name: "global-info",
docs: {
tenants: "tenants",
usageQuota: "usage_quota",
},
},
}

View File

@ -450,7 +450,7 @@ async function getScopedConfig(db, params) {
function generateNewUsageQuotaDoc() {
return {
_id: StaticDatabases.PLATFORM_INFO.docs.usageQuota,
_id: StaticDatabases.GLOBAL.docs.usageQuota,
quotaReset: Date.now() + 2592000000,
usageQuota: {
automationRuns: 0,

View File

@ -14,4 +14,5 @@ module.exports = {
cache: require("../cache"),
auth: require("../auth"),
constants: require("../constants"),
migrations: require("../migrations"),
}

View File

@ -1,5 +1,5 @@
const { DocumentTypes } = require("../db/constants")
const { getGlobalDB } = require("../tenancy")
const { getGlobalDB, getTenantId } = require("../tenancy")
exports.MIGRATION_DBS = {
GLOBAL_DB: "GLOBAL_DB",
@ -7,11 +7,13 @@ exports.MIGRATION_DBS = {
exports.MIGRATIONS = {
USER_EMAIL_VIEW_CASING: "user_email_view_casing",
QUOTAS_1: "quotas_1",
}
const DB_LOOKUP = {
[exports.MIGRATION_DBS.GLOBAL_DB]: [
exports.MIGRATIONS.USER_EMAIL_VIEW_CASING,
exports.MIGRATIONS.QUOTAS_1,
],
}
@ -27,6 +29,7 @@ exports.getMigrationsDoc = async db => {
}
exports.migrateIfRequired = async (migrationDb, migrationName, migrateFn) => {
const tenantId = getTenantId()
try {
let db
if (migrationDb === exports.MIGRATION_DBS.GLOBAL_DB) {
@ -47,15 +50,18 @@ exports.migrateIfRequired = async (migrationDb, migrationName, migrateFn) => {
return
}
console.log(`Performing migration: ${migrationName}`)
console.log(`[Tenant: ${tenantId}] Performing migration: ${migrationName}`)
await migrateFn()
console.log(`Migration complete: ${migrationName}`)
console.log(`[Tenant: ${tenantId}] Migration complete: ${migrationName}`)
// mark as complete
doc[migrationName] = Date.now()
await db.put(doc)
} catch (err) {
console.error(`Error performing migration: ${migrationName}: `, err)
console.error(
`[Tenant: ${tenantId}] Error performing migration: ${migrationName}: `,
err
)
throw err
}
}

View File

@ -1,7 +1,7 @@
{
"name": "@budibase/bbui",
"description": "A UI solution used in the different Budibase projects.",
"version": "1.0.44-alpha.9",
"version": "1.0.46-alpha.3",
"license": "MPL-2.0",
"svelte": "src/index.js",
"module": "dist/bbui.es.js",

View File

@ -1,6 +1,6 @@
{
"name": "@budibase/builder",
"version": "1.0.44-alpha.9",
"version": "1.0.46-alpha.3",
"license": "GPL-3.0",
"private": true,
"scripts": {
@ -65,10 +65,10 @@
}
},
"dependencies": {
"@budibase/bbui": "^1.0.44-alpha.9",
"@budibase/client": "^1.0.44-alpha.9",
"@budibase/bbui": "^1.0.46-alpha.3",
"@budibase/client": "^1.0.46-alpha.3",
"@budibase/colorpicker": "1.1.2",
"@budibase/string-templates": "^1.0.44-alpha.9",
"@budibase/string-templates": "^1.0.46-alpha.3",
"@sentry/browser": "5.19.1",
"@spectrum-css/page": "^3.0.1",
"@spectrum-css/vars": "^3.0.1",

View File

@ -6,6 +6,7 @@ const apiCall =
method =>
async (url, body, headers = { "Content-Type": "application/json" }) => {
headers["x-budibase-app-id"] = svelteGet(store).appId
headers["x-budibase-api-version"] = "1"
const json = headers["Content-Type"] === "application/json"
const resp = await fetch(url, {
method: method,

View File

@ -53,7 +53,9 @@
}
// Create table
const table = await tables.save(newTable)
let table
try {
table = await tables.save(newTable)
notifications.success(`Table ${name} created successfully.`)
analytics.captureEvent(Events.TABLE.CREATED, { name })
@ -63,6 +65,11 @@
? `./table/${table._id}`
: `../../table/${table._id}`
$goto(path)
} catch (e) {
notifications.error(e)
// reload in case the table was created
await tables.fetch()
}
}
</script>

View File

@ -55,8 +55,8 @@
<div>
<BindingBuilder
bind:customParams={parameters.queryParams}
bindings={query.parameters}
bind:bindableOptions={bindings}
queryBindings={query.parameters}
bind:bindings
/>
<IntegrationQueryEditor
height={200}

View File

@ -169,8 +169,8 @@
{#if getQueryParams(value).length > 0}
<BindingBuilder
bind:customParams={tmpQueryParams}
bindings={getQueryParams(value)}
bind:bindableOptions={bindings}
queryBindings={getQueryParams(value)}
bind:bindings
/>
{/if}
<IntegrationQueryEditor

View File

@ -7,27 +7,26 @@
import DrawerBindableInput from "components/common/bindings/DrawerBindableInput.svelte"
export let bindable = true
export let queryBindings = []
export let bindings = []
export let bindableOptions = []
export let customParams = {}
function newQueryBinding() {
bindings = [...bindings, {}]
queryBindings = [...queryBindings, {}]
}
$: console.log(bindings)
function deleteQueryBinding(idx) {
bindings.splice(idx, 1)
bindings = bindings
queryBindings.splice(idx, 1)
queryBindings = queryBindings
}
// This is necessary due to the way readable and writable bindings are stored.
// The readable binding in the UI gets converted to a UUID value that the client understands
// for parsing, then converted back so we can display it the readable form in the UI
function onBindingChange(param, valueToParse) {
customParams[param] = readableToRuntimeBinding(
bindableOptions,
valueToParse
)
customParams[param] = readableToRuntimeBinding(bindings, valueToParse)
}
</script>
@ -49,7 +48,7 @@
{/if}
</Body>
<div class="bindings" class:bindable>
{#each bindings as binding, idx}
{#each queryBindings as binding, idx}
<Input
placeholder="Binding Name"
thin
@ -69,10 +68,10 @@
thin
on:change={evt => onBindingChange(binding.name, evt.detail)}
value={runtimeToReadableBinding(
bindableOptions,
bindings,
customParams?.[binding.name]
)}
{bindableOptions}
{bindings}
/>
{:else}
<Icon hoverable name="Close" on:click={() => deleteQueryBinding(idx)} />

View File

@ -120,7 +120,7 @@
config={integrationInfo.extra}
/>
{/if}
<BindingBuilder bind:bindings={query.parameters} bindable={false} />
<BindingBuilder bind:queryBindings={query.parameters} bindable={false} />
{/if}
</div>
{#if shouldShowQueryConfig}

View File

@ -1,6 +1,6 @@
{
"name": "@budibase/cli",
"version": "1.0.44-alpha.9",
"version": "1.0.46-alpha.3",
"description": "Budibase CLI, for developers, self hosting and migrations.",
"main": "src/index.js",
"bin": {

View File

@ -1,6 +1,6 @@
{
"name": "@budibase/client",
"version": "1.0.44-alpha.9",
"version": "1.0.46-alpha.3",
"license": "MPL-2.0",
"module": "dist/budibase-client.js",
"main": "dist/budibase-client.js",
@ -19,9 +19,9 @@
"dev:builder": "rollup -cw"
},
"dependencies": {
"@budibase/bbui": "^1.0.44-alpha.9",
"@budibase/bbui": "^1.0.46-alpha.3",
"@budibase/standard-components": "^0.9.139",
"@budibase/string-templates": "^1.0.44-alpha.9",
"@budibase/string-templates": "^1.0.46-alpha.3",
"regexparam": "^1.3.0",
"shortid": "^2.2.15",
"svelte-spa-router": "^3.0.5"

View File

@ -110,12 +110,6 @@ export default class DataFetch {
*/
async getInitialData() {
const { datasource, filter, sortColumn, paginate } = this.options
const tableId = datasource?.tableId
// Ensure table ID exists
if (!tableId) {
return
}
// Fetch datasource definition and determine feature flags
const definition = await this.constructor.getDefinition(datasource)

View File

@ -1,7 +1,7 @@
{
"name": "@budibase/server",
"email": "hi@budibase.com",
"version": "1.0.44-alpha.9",
"version": "1.0.46-alpha.3",
"description": "Budibase Web Server",
"main": "src/index.ts",
"repository": {
@ -70,9 +70,9 @@
"license": "GPL-3.0",
"dependencies": {
"@apidevtools/swagger-parser": "^10.0.3",
"@budibase/backend-core": "^1.0.44-alpha.9",
"@budibase/client": "^1.0.44-alpha.9",
"@budibase/string-templates": "^1.0.44-alpha.9",
"@budibase/backend-core": "^1.0.46-alpha.3",
"@budibase/client": "^1.0.46-alpha.3",
"@budibase/string-templates": "^1.0.46-alpha.3",
"@bull-board/api": "^3.7.0",
"@bull-board/koa": "^3.7.0",
"@elastic/elasticsearch": "7.10.0",

View File

@ -8,6 +8,7 @@ const {
getTable,
handleDataImport,
} = require("./utils")
const usageQuota = require("../../../utilities/usageQuota")
exports.save = async function (ctx) {
const appId = ctx.appId
@ -119,6 +120,7 @@ exports.destroy = async function (ctx) {
})
)
await db.bulkDocs(rows.rows.map(row => ({ ...row.doc, _deleted: true })))
await usageQuota.update(usageQuota.Properties.ROW, -rows.rows.length)
// update linked rows
await linkRows.updateLinks({

View File

@ -16,6 +16,7 @@ const {
} = require("../../../integrations/utils")
const { getViews, saveView } = require("../view/utils")
const viewTemplate = require("../view/viewBuilder")
const usageQuota = require("../../../utilities/usageQuota")
exports.checkForColumnUpdates = async (db, oldTable, updatedTable) => {
let updatedRows = []
@ -112,7 +113,11 @@ exports.handleDataImport = async (appId, user, table, dataImport) => {
finalData.push(row)
}
await usageQuota.update(usageQuota.Properties.ROW, finalData.length, {
dryRun: true,
})
await db.bulkDocs(finalData)
await usageQuota.update(usageQuota.Properties.ROW, finalData.length)
let response = await db.put(table)
table._rev = response._rev
return table

View File

@ -1,6 +1,5 @@
const rowController = require("../../api/controllers/row")
const automationUtils = require("../automationUtils")
const env = require("../../environment")
const usage = require("../../utilities/usageQuota")
const { buildCtx } = require("./utils")
@ -83,10 +82,9 @@ exports.run = async function ({ inputs, appId, emitter }) {
inputs.row.tableId,
inputs.row
)
if (env.USE_QUOTAS) {
await usage.update(usage.Properties.ROW, 1)
}
await usage.update(usage.Properties.ROW, 1, { dryRun: true })
await rowController.save(ctx)
await usage.update(usage.Properties.ROW, 1)
return {
row: inputs.row,
response: ctx.body,

View File

@ -1,5 +1,4 @@
const rowController = require("../../api/controllers/row")
const env = require("../../environment")
const usage = require("../../utilities/usageQuota")
const { buildCtx } = require("./utils")
const automationUtils = require("../automationUtils")
@ -74,9 +73,7 @@ exports.run = async function ({ inputs, appId, emitter }) {
})
try {
if (env.isProd()) {
await usage.update(usage.Properties.ROW, -1)
}
await rowController.destroy(ctx)
return {
response: ctx.body,

View File

@ -38,6 +38,7 @@ module.exports = {
MINIO_ACCESS_KEY: process.env.MINIO_ACCESS_KEY,
MINIO_SECRET_KEY: process.env.MINIO_SECRET_KEY,
USE_QUOTAS: process.env.USE_QUOTAS,
EXCLUDE_QUOTAS_TENANTS: process.env.EXCLUDE_QUOTAS_TENANTS,
REDIS_URL: process.env.REDIS_URL,
REDIS_PASSWORD: process.env.REDIS_PASSWORD,
INTERNAL_API_KEY: process.env.INTERNAL_API_KEY,

View File

@ -1,11 +1,5 @@
jest.mock("../../db")
jest.mock("../../utilities/usageQuota")
jest.mock("../../environment", () => ({
isTest: () => true,
isProd: () => false,
isDev: () => true,
_set: () => {},
}))
jest.mock("@budibase/backend-core/tenancy", () => ({
getTenantId: () => "testing123"
}))
@ -29,9 +23,10 @@ class TestConfiguration {
},
req: {
method: "POST",
url: "/rows"
url: "/applications"
}
}
usageQuota.useQuotas = () => true
}
executeMiddleware() {
@ -113,7 +108,6 @@ describe("usageQuota middleware", () => {
it("calculates and persists the correct usage quota for the relevant action", async () => {
config.setUrl("/rows")
config.setProd(true)
await config.executeMiddleware()
@ -121,20 +115,20 @@ describe("usageQuota middleware", () => {
expect(config.next).toHaveBeenCalled()
})
it("calculates the correct file size from a file upload call and adds it to quota", async () => {
config.setUrl("/upload")
config.setProd(true)
config.setFiles([
{
size: 100
},
{
size: 10000
},
])
await config.executeMiddleware()
// it("calculates the correct file size from a file upload call and adds it to quota", async () => {
// config.setUrl("/upload")
// config.setProd(true)
// config.setFiles([
// {
// size: 100
// },
// {
// size: 10000
// },
// ])
// await config.executeMiddleware()
expect(usageQuota.update).toHaveBeenCalledWith("storage", 10100)
expect(config.next).toHaveBeenCalled()
})
// expect(usageQuota.update).toHaveBeenCalledWith("storage", 10100)
// expect(config.next).toHaveBeenCalled()
// })
})

View File

@ -1,14 +1,11 @@
const CouchDB = require("../db")
const usageQuota = require("../utilities/usageQuota")
const env = require("../environment")
const { getTenantId } = require("@budibase/backend-core/tenancy")
const { getUniqueRows } = require("../utilities/usageQuota/rows")
const {
isExternalTable,
isRowId: isExternalRowId,
} = require("../integrations/utils")
// tenants without limits
const EXCLUDED_TENANTS = ["bb", "default", "bbtest", "bbstaging"]
const migration = require("../migrations/usageQuotas")
// currently only counting new writes and deletes
const METHOD_MAP = {
@ -18,13 +15,13 @@ const METHOD_MAP = {
const DOMAIN_MAP = {
rows: usageQuota.Properties.ROW,
upload: usageQuota.Properties.UPLOAD,
views: usageQuota.Properties.VIEW,
users: usageQuota.Properties.USER,
// upload: usageQuota.Properties.UPLOAD, // doesn't work yet
// views: usageQuota.Properties.VIEW, // doesn't work yet
// users: usageQuota.Properties.USER, // doesn't work yet
applications: usageQuota.Properties.APPS,
// this will not be updated by endpoint calls
// instead it will be updated by triggerInfo
automationRuns: usageQuota.Properties.AUTOMATION,
// automationRuns: usageQuota.Properties.AUTOMATION, // doesn't work yet
}
function getProperty(url) {
@ -36,10 +33,7 @@ function getProperty(url) {
}
module.exports = async (ctx, next) => {
const tenantId = getTenantId()
// if in development or a self hosted cloud usage quotas should not be executed
if (env.isDev() || env.SELF_HOSTED || EXCLUDED_TENANTS.includes(tenantId)) {
if (!usageQuota.useQuotas()) {
return next()
}
@ -80,9 +74,93 @@ module.exports = async (ctx, next) => {
usage = files.map(file => file.size).reduce((total, size) => total + size)
}
try {
await usageQuota.update(property, usage)
return next()
await migration.run()
await performRequest(ctx, next, property, usage)
} catch (err) {
ctx.throw(400, err)
}
}
const performRequest = async (ctx, next, property, usage) => {
const usageContext = {
skipNext: false,
skipUsage: false,
[usageQuota.Properties.APPS]: {},
}
if (usage === -1) {
if (PRE_DELETE[property]) {
await PRE_DELETE[property](ctx, usageContext)
}
} else {
if (PRE_CREATE[property]) {
await PRE_CREATE[property](ctx, usageContext)
}
}
// run the request
if (!usageContext.skipNext) {
await usageQuota.update(property, usage, { dryRun: true })
await next()
}
if (usage === -1) {
if (POST_DELETE[property]) {
await POST_DELETE[property](ctx, usageContext)
}
} else {
if (POST_CREATE[property]) {
await POST_CREATE[property](ctx, usageContext)
}
}
// update the usage
if (!usageContext.skipUsage) {
await usageQuota.update(property, usage)
}
}
const appPreDelete = async (ctx, usageContext) => {
if (ctx.query.unpublish) {
// don't run usage decrement for unpublish
usageContext.skipUsage = true
return
}
// store the row count to delete
const rows = await getUniqueRows([ctx.appId])
if (rows.length) {
usageContext[usageQuota.Properties.APPS] = { rowCount: rows.length }
}
}
const appPostDelete = async (ctx, usageContext) => {
// delete the app rows from usage
const rowCount = usageContext[usageQuota.Properties.APPS].rowCount
if (rowCount) {
await usageQuota.update(usageQuota.Properties.ROW, -rowCount)
}
}
const appPostCreate = async ctx => {
// app import & template creation
if (ctx.request.body.useTemplate === "true") {
const rows = await getUniqueRows([ctx.response.body.appId])
const rowCount = rows ? rows.length : 0
await usageQuota.update(usageQuota.Properties.ROW, rowCount)
}
}
const PRE_DELETE = {
[usageQuota.Properties.APPS]: appPreDelete,
}
const POST_DELETE = {
[usageQuota.Properties.APPS]: appPostDelete,
}
const PRE_CREATE = {}
const POST_CREATE = {
[usageQuota.Properties.APPS]: appPostCreate,
}

View File

@ -0,0 +1,27 @@
const env = require("../../../environment")
const TestConfig = require("../../../tests/utilities/TestConfiguration")
const syncApps = jest.fn()
const syncRows = jest.fn()
jest.mock("../../usageQuotas/syncApps", () => ({ run: syncApps }) )
jest.mock("../../usageQuotas/syncRows", () => ({ run: syncRows }) )
const migrations = require("../../usageQuotas")
describe("run", () => {
let config = new TestConfig(false)
beforeEach(async () => {
await config.init()
env._set("USE_QUOTAS", 1)
})
afterAll(config.end)
it("runs the required migrations", async () => {
await migrations.run()
expect(syncApps).toHaveBeenCalledTimes(1)
expect(syncRows).toHaveBeenCalledTimes(1)
})
})

View File

@ -0,0 +1,37 @@
const { getGlobalDB } = require("@budibase/backend-core/tenancy")
const TestConfig = require("../../../tests/utilities/TestConfiguration")
const { getUsageQuotaDoc, update, Properties } = require("../../../utilities/usageQuota")
const syncApps = require("../../usageQuotas/syncApps")
const env = require("../../../environment")
describe("syncApps", () => {
let config = new TestConfig(false)
beforeEach(async () => {
await config.init()
env._set("USE_QUOTAS", 1)
})
afterAll(config.end)
it("runs successfully", async () => {
// create the usage quota doc and mock usages
const db = getGlobalDB()
await getUsageQuotaDoc(db)
await update(Properties.APPS, 3)
let usageDoc = await getUsageQuotaDoc(db)
expect(usageDoc.usageQuota.apps).toEqual(3)
// create an extra app to test the migration
await config.createApp("quota-test")
// migrate
await syncApps.run()
// assert the migration worked
usageDoc = await getUsageQuotaDoc(db)
expect(usageDoc.usageQuota.apps).toEqual(2)
})
})

View File

@ -0,0 +1,43 @@
const { getGlobalDB } = require("@budibase/backend-core/tenancy")
const TestConfig = require("../../../tests/utilities/TestConfiguration")
const { getUsageQuotaDoc, update, Properties } = require("../../../utilities/usageQuota")
const syncRows = require("../../usageQuotas/syncRows")
const env = require("../../../environment")
describe("syncRows", () => {
let config = new TestConfig(false)
beforeEach(async () => {
await config.init()
env._set("USE_QUOTAS", 1)
})
afterAll(config.end)
it("runs successfully", async () => {
// create the usage quota doc and mock usages
const db = getGlobalDB()
await getUsageQuotaDoc(db)
await update(Properties.ROW, 300)
let usageDoc = await getUsageQuotaDoc(db)
expect(usageDoc.usageQuota.rows).toEqual(300)
// app 1
await config.createTable()
await config.createRow()
// app 2
await config.createApp()
await config.createTable()
await config.createRow()
await config.createRow()
// migrate
await syncRows.run()
// assert the migration worked
usageDoc = await getUsageQuotaDoc(db)
expect(usageDoc.usageQuota.rows).toEqual(3)
})
})

View File

@ -0,0 +1,24 @@
const {
MIGRATIONS,
MIGRATION_DBS,
migrateIfRequired,
} = require("@budibase/backend-core/migrations")
const { useQuotas } = require("../../utilities/usageQuota")
const syncApps = require("./syncApps")
const syncRows = require("./syncRows")
exports.run = async () => {
if (!useQuotas()) {
return
}
// Jan 2022
await migrateIfRequired(
MIGRATION_DBS.GLOBAL_DB,
MIGRATIONS.QUOTAS_1,
async () => {
await syncApps.run()
await syncRows.run()
}
)
}

View File

@ -0,0 +1,18 @@
const { getGlobalDB, getTenantId } = require("@budibase/backend-core/tenancy")
const { getAllApps } = require("@budibase/backend-core/db")
const CouchDB = require("../../db")
const { getUsageQuotaDoc } = require("../../utilities/usageQuota")
exports.run = async () => {
const db = getGlobalDB()
// get app count
const devApps = await getAllApps(CouchDB, { dev: true })
const appCount = devApps ? devApps.length : 0
// sync app count
const tenantId = getTenantId()
console.log(`[Tenant: ${tenantId}] Syncing app count: ${appCount}`)
const usageDoc = await getUsageQuotaDoc(db)
usageDoc.usageQuota.apps = appCount
await db.put(usageDoc)
}

View File

@ -0,0 +1,21 @@
const { getGlobalDB, getTenantId } = require("@budibase/backend-core/tenancy")
const { getAllApps } = require("@budibase/backend-core/db")
const CouchDB = require("../../db")
const { getUsageQuotaDoc } = require("../../utilities/usageQuota")
const { getUniqueRows } = require("../../utilities/usageQuota/rows")
exports.run = async () => {
const db = getGlobalDB()
// get all rows in all apps
const allApps = await getAllApps(CouchDB, { all: true })
const appIds = allApps ? allApps.map(app => app.appId) : []
const rows = await getUniqueRows(appIds)
const rowCount = rows ? rows.length : 0
// sync row count
const tenantId = getTenantId()
console.log(`[Tenant: ${tenantId}] Syncing row count: ${rowCount}`)
const usageDoc = await getUsageQuotaDoc(db)
usageDoc.usageQuota.rows = rowCount
await db.put(usageDoc)
}

View File

@ -0,0 +1,72 @@
const getTenantId = jest.fn()
jest.mock("@budibase/backend-core/tenancy", () => ({
getTenantId
}))
const usageQuota = require("../../usageQuota")
const env = require("../../../environment")
class TestConfiguration {
constructor() {
this.enableQuotas()
}
enableQuotas = () => {
env.USE_QUOTAS = 1
}
disableQuotas = () => {
env.USE_QUOTAS = null
}
setTenantId = (tenantId) => {
getTenantId.mockReturnValue(tenantId)
}
setExcludedTenants = (tenants) => {
env.EXCLUDE_QUOTAS_TENANTS = tenants
}
reset = () => {
this.disableQuotas()
this.setExcludedTenants(null)
}
}
describe("usageQuota", () => {
let config
beforeEach(() => {
config = new TestConfiguration()
})
afterEach(() => {
config.reset()
})
describe("useQuotas", () => {
it("works when no settings have been provided", () => {
config.reset()
expect(usageQuota.useQuotas()).toBe(false)
})
it("honours USE_QUOTAS setting", () => {
config.disableQuotas()
expect(usageQuota.useQuotas()).toBe(false)
config.enableQuotas()
expect(usageQuota.useQuotas()).toBe(true)
})
it("honours EXCLUDE_QUOTAS_TENANTS setting", () => {
config.setTenantId("test")
// tenantId is in the list
config.setExcludedTenants("test, test2, test2")
expect(usageQuota.useQuotas()).toBe(false)
config.setExcludedTenants("test,test2,test2")
expect(usageQuota.useQuotas()).toBe(false)
// tenantId is not in the list
config.setTenantId("other")
expect(usageQuota.useQuotas()).toBe(true)
})
})
})

View File

@ -1,73 +0,0 @@
const env = require("../environment")
const { getGlobalDB } = require("@budibase/backend-core/tenancy")
const {
StaticDatabases,
generateNewUsageQuotaDoc,
} = require("@budibase/backend-core/db")
function getNewQuotaReset() {
return Date.now() + 2592000000
}
exports.Properties = {
ROW: "rows",
UPLOAD: "storage",
VIEW: "views",
USER: "users",
AUTOMATION: "automationRuns",
APPS: "apps",
EMAILS: "emails",
}
async function getUsageQuotaDoc(db) {
let quota
try {
quota = await db.get(StaticDatabases.PLATFORM_INFO.docs.usageQuota)
} catch (err) {
// doc doesn't exist. Create it
quota = await db.post(generateNewUsageQuotaDoc())
}
return quota
}
/**
* Given a specified tenantId this will add to the usage object for the specified property.
* @param {string} property The property which is to be added to (within the nested usageQuota object).
* @param {number} usage The amount (this can be negative) to adjust the number by.
* @returns {Promise<void>} When this completes the API key will now be up to date - the quota period may have
* also been reset after this call.
*/
exports.update = async (property, usage) => {
if (!env.USE_QUOTAS) {
return
}
try {
const db = getGlobalDB()
const quota = await getUsageQuotaDoc(db)
// Check if the quota needs reset
if (Date.now() >= quota.quotaReset) {
quota.quotaReset = getNewQuotaReset()
for (let prop of Object.keys(quota.usageQuota)) {
quota.usageQuota[prop] = 0
}
}
// increment the quota
quota.usageQuota[property] += usage
if (quota.usageQuota[property] > quota.usageLimits[property]) {
throw new Error(
`You have exceeded your usage quota of ${quota.usageLimits[property]} ${property}.`
)
}
// update the usage quotas
await db.put(quota)
} catch (err) {
console.error(`Error updating usage quotas for ${property}`, err)
throw err
}
}

View File

@ -0,0 +1,92 @@
const env = require("../../environment")
const { getGlobalDB, getTenantId } = require("@budibase/backend-core/tenancy")
const {
StaticDatabases,
generateNewUsageQuotaDoc,
} = require("@budibase/backend-core/db")
exports.useQuotas = () => {
// check if quotas are enabled
if (env.USE_QUOTAS) {
// check if there are any tenants without limits
if (env.EXCLUDE_QUOTAS_TENANTS) {
const excludedTenants = env.EXCLUDE_QUOTAS_TENANTS.replace(
/\s/g,
""
).split(",")
const tenantId = getTenantId()
if (excludedTenants.includes(tenantId)) {
return false
}
}
return true
}
return false
}
exports.Properties = {
ROW: "rows",
UPLOAD: "storage", // doesn't work yet
VIEW: "views", // doesn't work yet
USER: "users", // doesn't work yet
AUTOMATION: "automationRuns", // doesn't work yet
APPS: "apps",
EMAILS: "emails", // doesn't work yet
}
exports.getUsageQuotaDoc = async db => {
let quota
try {
quota = await db.get(StaticDatabases.GLOBAL.docs.usageQuota)
} catch (err) {
// doc doesn't exist. Create it
quota = generateNewUsageQuotaDoc()
const response = await db.put(quota)
quota._rev = response.rev
}
return quota
}
/**
* Given a specified tenantId this will add to the usage object for the specified property.
* @param {string} property The property which is to be added to (within the nested usageQuota object).
* @param {number} usage The amount (this can be negative) to adjust the number by.
* @returns {Promise<void>} When this completes the API key will now be up to date - the quota period may have
* also been reset after this call.
*/
exports.update = async (property, usage, opts = { dryRun: false }) => {
if (!exports.useQuotas()) {
return
}
try {
const db = getGlobalDB()
const quota = await exports.getUsageQuotaDoc(db)
// increment the quota
quota.usageQuota[property] += usage
if (
quota.usageQuota[property] > quota.usageLimits[property] &&
usage > 0 // allow for decrementing usage when the quota is already exceeded
) {
throw new Error(
`You have exceeded your usage quota of ${quota.usageLimits[property]} ${property}.`
)
}
if (quota.usageQuota[property] < 0) {
// never go negative if the quota has previously been exceeded
quota.usageQuota[property] = 0
}
// update the usage quotas
if (!opts.dryRun) {
await db.put(quota)
}
} catch (err) {
console.error(`Error updating usage quotas for ${property}`, err)
throw err
}
}

View File

@ -0,0 +1,74 @@
const { getRowParams, USER_METDATA_PREFIX } = require("../../db/utils")
const CouchDB = require("../../db")
const { isDevAppID, getDevelopmentAppID } = require("@budibase/backend-core/db")
const ROW_EXCLUSIONS = [USER_METDATA_PREFIX]
const getAppPairs = appIds => {
// collect the app ids into dev / prod pairs
// keyed by the dev app id
const pairs = {}
for (let appId of appIds) {
const devId = getDevelopmentAppID(appId)
if (!pairs[devId]) {
pairs[devId] = {}
}
if (isDevAppID(appId)) {
pairs[devId].devId = appId
} else {
pairs[devId].prodId = appId
}
}
return pairs
}
const getAppRows = async appId => {
const appDb = new CouchDB(appId)
const response = await appDb.allDocs(
getRowParams(null, null, {
include_docs: false,
})
)
return response.rows
.map(r => r.id)
.filter(id => {
for (let exclusion of ROW_EXCLUSIONS) {
if (id.startsWith(exclusion)) {
return false
}
}
return true
})
}
/**
* Return a set of all rows in the given app ids.
* The returned rows will be unique on a per dev/prod app basis.
* Rows duplicates may exist across apps due to data import so they are not filtered out.
*/
exports.getUniqueRows = async appIds => {
let uniqueRows = []
const pairs = getAppPairs(appIds)
for (let pair of Object.values(pairs)) {
let appRows = []
for (let appId of [pair.devId, pair.prodId]) {
if (!appId) {
continue
}
try {
appRows.push(await getAppRows(appId))
} catch (e) {
console.error(e)
// don't error out if we can't count the app rows, just continue
}
}
// ensure uniqueness on a per app pair basis
// this can't be done on all rows because app import results in
// duplicate row ids across apps
uniqueRows = uniqueRows.concat(...new Set(appRows))
}
return uniqueRows
}

View File

@ -0,0 +1,18 @@
// UNUSED CODE
// Preserved for future use
/* eslint-disable no-unused-vars */
function getNewQuotaReset() {
return Date.now() + 2592000000
}
function resetQuotasIfRequired(quota) {
// Check if the quota needs reset
if (Date.now() >= quota.quotaReset) {
quota.quotaReset = getNewQuotaReset()
for (let prop of Object.keys(quota.usageQuota)) {
quota.usageQuota[prop] = 0
}
}
}

View File

@ -1,6 +1,6 @@
{
"name": "@budibase/string-templates",
"version": "1.0.44-alpha.9",
"version": "1.0.46-alpha.3",
"description": "Handlebars wrapper for Budibase templating.",
"main": "src/index.cjs",
"module": "dist/bundle.mjs",

View File

@ -1,5 +1,6 @@
const { atob } = require("../utilities")
const { cloneDeep } = require("lodash/fp")
const { LITERAL_MARKER } = require("../helpers/constants")
// The method of executing JS scripts depends on the bundle being built.
// This setter is used in the entrypoint (either index.cjs or index.mjs).
@ -46,8 +47,9 @@ module.exports.processJS = (handlebars, context) => {
$: path => getContextValue(path, cloneDeep(context)),
}
// Create a sandbox with out context and run the JS
return runJS(js, sandboxContext)
// Create a sandbox with our context and run the JS
const res = { data: runJS(js, sandboxContext) }
return `{{${LITERAL_MARKER} js_result-${JSON.stringify(res)}}}`
} catch (error) {
return "Error while executing JS"
}

View File

@ -36,6 +36,11 @@ module.exports.processors = [
return value === "true"
case "object":
return JSON.parse(value)
case "js_result":
// We use the literal helper to process the result of JS expressions
// as we want to be able to return any types.
// We wrap the value in an abject to be able to use undefined properly.
return JSON.parse(value).data
}
return value
}),

View File

@ -7,7 +7,7 @@ const processJS = (js, context) => {
describe("Test the JavaScript helper", () => {
it("should execute a simple expression", () => {
const output = processJS(`return 1 + 2`)
expect(output).toBe("3")
expect(output).toBe(3)
})
it("should be able to use primitive bindings", () => {
@ -50,6 +50,52 @@ describe("Test the JavaScript helper", () => {
expect(output).toBe("shazbat")
})
it("should be able to return an object", () => {
const output = processJS(`return $("foo")`, {
foo: {
bar: {
baz: "shazbat",
},
},
})
expect(output.bar.baz).toBe("shazbat")
})
it("should be able to return an array", () => {
const output = processJS(`return $("foo")`, {
foo: ["a", "b", "c"],
})
expect(output[2]).toBe("c")
})
it("should be able to return null", () => {
const output = processJS(`return $("foo")`, {
foo: null,
})
expect(output).toBe(null)
})
it("should be able to return undefined", () => {
const output = processJS(`return $("foo")`, {
foo: undefined,
})
expect(output).toBe(undefined)
})
it("should be able to return 0", () => {
const output = processJS(`return $("foo")`, {
foo: 0,
})
expect(output).toBe(0)
})
it("should be able to return an empty string", () => {
const output = processJS(`return $("foo")`, {
foo: "",
})
expect(output).toBe("")
})
it("should be able to use a deep array binding", () => {
const output = processJS(`return $("foo.0.bar")`, {
foo: [

View File

@ -1,7 +1,7 @@
{
"name": "@budibase/worker",
"email": "hi@budibase.com",
"version": "1.0.44-alpha.9",
"version": "1.0.46-alpha.3",
"description": "Budibase background service",
"main": "src/index.js",
"repository": {
@ -29,8 +29,8 @@
"author": "Budibase",
"license": "GPL-3.0",
"dependencies": {
"@budibase/backend-core": "^1.0.44-alpha.9",
"@budibase/string-templates": "^1.0.44-alpha.9",
"@budibase/backend-core": "^1.0.46-alpha.3",
"@budibase/string-templates": "^1.0.46-alpha.3",
"@koa/router": "^8.0.0",
"@sentry/node": "^6.0.0",
"@techpass/passport-openidconnect": "^0.3.0",

View File

@ -73,16 +73,14 @@ exports.adminUser = async ctx => {
if (!env.SELF_HOSTED) {
// could be a scenario where it exists, make sure its clean
try {
const usageQuota = await db.get(
StaticDatabases.PLATFORM_INFO.docs.usageQuota
)
const usageQuota = await db.get(StaticDatabases.GLOBAL.docs.usageQuota)
if (usageQuota) {
await db.remove(usageQuota._id, usageQuota._rev)
}
} catch (err) {
// don't worry about errors
}
await db.post(generateNewUsageQuotaDoc())
await db.put(generateNewUsageQuotaDoc())
}
if (response.rows.some(row => row.doc.admin)) {