Merge branch 'develop' of github.com:Budibase/budibase into cloud-limits

This commit is contained in:
Martin McKeaveney 2021-09-27 12:53:15 +01:00
commit 3b49866825
25 changed files with 178 additions and 71 deletions

View File

@ -1,5 +1,5 @@
{
"version": "0.9.140-alpha.8",
"version": "0.9.140-alpha.11",
"npmClient": "yarn",
"packages": [
"packages/*"

View File

@ -1,6 +1,6 @@
{
"name": "@budibase/auth",
"version": "0.9.140-alpha.8",
"version": "0.9.140-alpha.11",
"description": "Authentication middlewares for budibase builder and apps",
"main": "src/index.js",
"author": "Budibase",

View File

@ -1,7 +1,7 @@
{
"name": "@budibase/bbui",
"description": "A UI solution used in the different Budibase projects.",
"version": "0.9.140-alpha.8",
"version": "0.9.140-alpha.11",
"license": "AGPL-3.0",
"svelte": "src/index.js",
"module": "dist/bbui.es.js",

View File

@ -1,6 +1,6 @@
{
"name": "@budibase/builder",
"version": "0.9.140-alpha.8",
"version": "0.9.140-alpha.11",
"license": "AGPL-3.0",
"private": true,
"scripts": {
@ -65,10 +65,10 @@
}
},
"dependencies": {
"@budibase/bbui": "^0.9.140-alpha.8",
"@budibase/client": "^0.9.140-alpha.8",
"@budibase/bbui": "^0.9.140-alpha.11",
"@budibase/client": "^0.9.140-alpha.11",
"@budibase/colorpicker": "1.1.2",
"@budibase/string-templates": "^0.9.140-alpha.8",
"@budibase/string-templates": "^0.9.140-alpha.11",
"@sentry/browser": "5.19.1",
"@spectrum-css/page": "^3.0.1",
"@spectrum-css/vars": "^3.0.1",

View File

@ -443,10 +443,9 @@ function bindingReplacement(bindableProperties, textWithBindings, convertTo) {
for (let from of convertFromProps) {
if (shouldReplaceBinding(newBoundValue, from, convertTo)) {
const binding = bindableProperties.find(el => el[convertFrom] === from)
newBoundValue = newBoundValue.replace(
new RegExp(from, "gi"),
binding[convertTo]
)
while (newBoundValue.includes(from)) {
newBoundValue = newBoundValue.replace(from, binding[convertTo])
}
}
}
result = result.replace(boundValue, newBoundValue)

View File

@ -1,8 +1,9 @@
<script>
import { onMount } from "svelte"
import { get } from "svelte/store"
import { goto } from "@roxi/routify"
import { BUDIBASE_INTERNAL_DB } from "constants"
import { database, datasources, queries } from "stores/backend"
import { database, datasources, queries, tables } from "stores/backend"
import EditDatasourcePopover from "./popovers/EditDatasourcePopover.svelte"
import EditQueryPopover from "./popovers/EditQueryPopover.svelte"
import NavItem from "components/common/NavItem.svelte"
@ -10,6 +11,13 @@
import ICONS from "./icons"
let openDataSources = []
$: enrichedDataSources = $datasources.list.map(datasource => ({
...datasource,
open:
openDataSources.includes(datasource._id) ||
containsActiveTable(datasource),
selected: $datasources.selected === datasource._id,
}))
function selectDatasource(datasource) {
toggleNode(datasource)
@ -35,16 +43,28 @@
datasources.fetch()
queries.fetch()
})
const containsActiveTable = datasource => {
const activeTableId = get(tables).selected?._id
if (!datasource.entities) {
return false
}
let tableOptions = datasource.entities
if (!Array.isArray(tableOptions)) {
tableOptions = Object.values(tableOptions)
}
return tableOptions.find(x => x._id === activeTableId) != null
}
</script>
{#if $database?._id}
<div class="hierarchy-items-container">
{#each $datasources.list as datasource, idx}
{#each enrichedDataSources as datasource, idx}
<NavItem
border={idx > 0}
text={datasource.name}
opened={openDataSources.includes(datasource._id)}
selected={$datasources.selected === datasource._id}
opened={datasource.open}
selected={datasource.selected}
withArrow={true}
on:click={() => selectDatasource(datasource)}
on:iconClick={() => toggleNode(datasource)}
@ -61,7 +81,7 @@
{/if}
</NavItem>
{#if openDataSources.includes(datasource._id)}
{#if datasource.open}
<TableNavigator sourceId={datasource._id} />
{#each $queries.list.filter(query => query.datasourceId === datasource._id) as query}
<NavItem

View File

@ -0,0 +1,35 @@
<script>
import { Select, Label } from "@budibase/bbui"
import { currentAsset, store } from "builderStore"
import { getActionProviderComponents } from "builderStore/dataBinding"
export let parameters
$: actionProviders = getActionProviderComponents(
$currentAsset,
$store.selectedComponentId,
"RefreshDataProvider"
)
</script>
<div class="root">
<Label small>Data Provider</Label>
<Select
bind:value={parameters.componentId}
options={actionProviders}
getOptionLabel={x => x._instanceName}
getOptionValue={x => x._id}
/>
</div>
<style>
.root {
display: grid;
column-gap: var(--spacing-l);
row-gap: var(--spacing-s);
grid-template-columns: 70px 1fr;
align-items: center;
max-width: 400px;
margin: 0 auto;
}
</style>

View File

@ -12,6 +12,7 @@ import ClearForm from "./ClearForm.svelte"
import CloseScreenModal from "./CloseScreenModal.svelte"
import ChangeFormStep from "./ChangeFormStep.svelte"
import UpdateStateStep from "./UpdateState.svelte"
import RefreshDataProvider from "./RefreshDataProvider.svelte"
// Defines which actions are available to configure in the front end.
// Unfortunately the "name" property is used as the identifier so please don't
@ -62,6 +63,10 @@ export const getAvailableActions = () => {
name: "Change Form Step",
component: ChangeFormStep,
},
{
name: "Refresh Data Provider",
component: RefreshDataProvider,
},
]
if (get(store).clientFeatures?.state) {

View File

@ -0,0 +1,7 @@
<script>
import { datasources } from "stores/backend"
datasources.select("bb_internal")
</script>
<slot />

View File

@ -1,4 +1,4 @@
import { writable } from "svelte/store"
import { writable, get } from "svelte/store"
import { queries, tables, views } from "./"
import api from "../../builderStore/api"
@ -8,7 +8,8 @@ export const INITIAL_DATASOURCE_VALUES = {
}
export function createDatasourcesStore() {
const { subscribe, update, set } = writable(INITIAL_DATASOURCE_VALUES)
const store = writable(INITIAL_DATASOURCE_VALUES)
const { subscribe, update, set } = store
return {
subscribe,
@ -21,7 +22,15 @@ export function createDatasourcesStore() {
fetch: async () => {
const response = await api.get(`/api/datasources`)
const json = await response.json()
update(state => ({ ...state, list: json, selected: null }))
// Clear selected if it no longer exists, otherwise keep it
const selected = get(store).selected
let nextSelected = null
if (selected && json.find(source => source._id === selected)) {
nextSelected = selected
}
update(state => ({ ...state, list: json, selected: nextSelected }))
return json
},
select: async datasourceId => {

View File

@ -1,6 +1,6 @@
{
"name": "@budibase/cli",
"version": "0.9.140-alpha.8",
"version": "0.9.140-alpha.11",
"description": "Budibase CLI, for developers, self hosting and migrations.",
"main": "src/index.js",
"bin": {

View File

@ -2389,6 +2389,7 @@
"icon": "Data",
"illegalChildren": ["section"],
"hasChildren": true,
"actions": ["RefreshDatasource"],
"settings": [
{
"type": "dataSource",

View File

@ -1,6 +1,6 @@
{
"name": "@budibase/client",
"version": "0.9.140-alpha.8",
"version": "0.9.140-alpha.11",
"license": "MPL-2.0",
"module": "dist/budibase-client.js",
"main": "dist/budibase-client.js",
@ -19,9 +19,9 @@
"dev:builder": "rollup -cw"
},
"dependencies": {
"@budibase/bbui": "^0.9.140-alpha.8",
"@budibase/bbui": "^0.9.140-alpha.11",
"@budibase/standard-components": "^0.9.139",
"@budibase/string-templates": "^0.9.140-alpha.8",
"@budibase/string-templates": "^0.9.140-alpha.11",
"regexparam": "^1.3.0",
"shortid": "^2.2.15",
"svelte-spa-router": "^3.0.5"

View File

@ -88,7 +88,7 @@ const validateFormHandler = async (action, context) => {
)
}
const refreshDatasourceHandler = async (action, context) => {
const refreshDataProviderHandler = async (action, context) => {
return await executeActionHandler(
context,
action.parameters.componentId,
@ -139,7 +139,7 @@ const handlerMap = {
["Execute Query"]: queryExecutionHandler,
["Trigger Automation"]: triggerAutomationHandler,
["Validate Form"]: validateFormHandler,
["Refresh Datasource"]: refreshDatasourceHandler,
["Refresh Data Provider"]: refreshDataProviderHandler,
["Log Out"]: logoutHandler,
["Clear Form"]: clearFormHandler,
["Close Screen Modal"]: closeScreenModalHandler,

View File

@ -1,7 +1,7 @@
{
"name": "@budibase/server",
"email": "hi@budibase.com",
"version": "0.9.140-alpha.8",
"version": "0.9.140-alpha.11",
"description": "Budibase Web Server",
"main": "src/index.js",
"repository": {
@ -62,9 +62,9 @@
"author": "Budibase",
"license": "AGPL-3.0-or-later",
"dependencies": {
"@budibase/auth": "^0.9.140-alpha.8",
"@budibase/client": "^0.9.140-alpha.8",
"@budibase/string-templates": "^0.9.140-alpha.8",
"@budibase/auth": "^0.9.140-alpha.11",
"@budibase/client": "^0.9.140-alpha.11",
"@budibase/string-templates": "^0.9.140-alpha.11",
"@elastic/elasticsearch": "7.10.0",
"@koa/router": "8.0.0",
"@sendgrid/mail": "7.1.1",

View File

@ -15,7 +15,7 @@ services:
- ./init.sql:/docker-entrypoint-initdb.d/init.sql
pgadmin:
container_name: pgadmin
container_name: pgadmin-pg
image: dpage/pgadmin4
restart: always
environment:

View File

@ -230,7 +230,12 @@ exports.create = async function (ctx) {
const response = await db.put(newApplication, { force: true })
newApplication._rev = response.rev
await createEmptyAppPackage(ctx, newApplication)
// Only create the default home screens and layout if we aren't importing
// an app
if (useTemplate !== "true") {
await createEmptyAppPackage(ctx, newApplication)
}
/* istanbul ignore next */
if (!env.isTest()) {
await createApp(appId)

View File

@ -544,6 +544,9 @@ module External {
extra: {
idFilter: buildFilters(id || generateIdForRow(row, table), {}, table),
},
meta: {
table,
}
}
// can't really use response right now
const response = await makeExternalQuery(appId, json)

View File

@ -94,7 +94,8 @@ describe("/datasources", () => {
.expect(200)
// this is mock data, can't test it
expect(res.body).toBeDefined()
expect(pg.queryMock).toHaveBeenCalledWith(`select "users"."name" as "users.name", "users"."age" as "users.age" from "users" where "users"."name" ilike $1 limit $2`, ["John%", 5000])
const expSql = `select "users"."name" as "users.name", "users"."age" as "users.age" from (select * from "users" where "users"."name" ilike $1 limit $2) as "users"`
expect(pg.queryMock).toHaveBeenCalledWith(expSql, ["John%", 5000])
})
})

View File

@ -1,3 +1,5 @@
import {Table} from "./common";
export enum Operation {
CREATE = "CREATE",
READ = "READ",
@ -136,6 +138,9 @@ export interface QueryJson {
sort?: SortJson
paginate?: PaginationJson
body?: object
meta?: {
table?: Table,
}
extra?: {
idFilter?: SearchFilters
}

View File

@ -1,7 +1,5 @@
import { Knex, knex } from "knex"
const BASE_LIMIT = 5000
// if requesting a single row then need to up the limit for the sake of joins
const SINGLE_ROW_LIMIT = 100
import {
QueryJson,
SearchFilters,
@ -146,46 +144,48 @@ function buildCreate(
function buildRead(knex: Knex, json: QueryJson, limit: number): KnexQuery {
let { endpoint, resource, filters, sort, paginate, relationships } = json
const tableName = endpoint.entityId
let query: KnexQuery = knex(tableName)
// select all if not specified
if (!resource) {
resource = { fields: [] }
}
let selectStatement: string|string[] = "*"
// handle select
if (resource.fields && resource.fields.length > 0) {
// select the resources as the format "table.columnName" - this is what is provided
// by the resource builder further up
query = query.select(resource.fields.map(field => `${field} as ${field}`))
} else {
query = query.select("*")
selectStatement = resource.fields.map(field => `${field} as ${field}`)
}
let foundLimit = limit || BASE_LIMIT
// handle pagination
let foundOffset: number | null = null
if (paginate && paginate.page && paginate.limit) {
// @ts-ignore
const page = paginate.page <= 1 ? 0 : paginate.page - 1
const offset = page * paginate.limit
foundLimit = paginate.limit
foundOffset = offset
} else if (paginate && paginate.limit) {
foundLimit = paginate.limit
}
// start building the query
let query: KnexQuery = knex(tableName).limit(foundLimit)
if (foundOffset) {
query = query.offset(foundOffset)
}
// handle where
query = addFilters(tableName, query, filters)
// handle join
query = addRelationships(query, tableName, relationships)
// handle sorting
if (sort) {
for (let [key, value] of Object.entries(sort)) {
const direction = value === SortDirection.ASCENDING ? "asc" : "desc"
query = query.orderBy(key, direction)
}
}
let foundLimit = limit || BASE_LIMIT
// handle pagination
if (paginate && paginate.page && paginate.limit) {
query = addFilters(tableName, query, filters)
// @ts-ignore
let preQuery: KnexQuery = knex({
// @ts-ignore
const page = paginate.page <= 1 ? 0 : paginate.page - 1
const offset = page * paginate.limit
foundLimit = paginate.limit
query = query.offset(offset)
} else if (paginate && paginate.limit) {
foundLimit = paginate.limit
}
if (foundLimit === 1) {
foundLimit = SINGLE_ROW_LIMIT
}
query = query.limit(foundLimit)
return query
[tableName]: query,
}).select(selectStatement)
// handle joins
return addRelationships(preQuery, tableName, relationships)
}
function buildUpdate(

View File

@ -108,7 +108,7 @@ module MySQLModule {
client: any,
query: SqlQuery,
connect: boolean = true
): Promise<any[]> {
): Promise<any[]|any> {
// Node MySQL is callback based, so we must wrap our call in a promise
return new Promise((resolve, reject) => {
if (connect) {
@ -242,6 +242,23 @@ module MySQLModule {
return internalQuery(this.client, input, false)
}
// when creating if an ID has been inserted need to make sure
// the id filter is enriched with it before trying to retrieve the row
checkLookupKeys(results: any, json: QueryJson) {
if (!results?.insertId || !json.meta?.table || !json.meta.table.primary) {
return json
}
const primaryKey = json.meta.table.primary?.[0]
json.extra = {
idFilter: {
equal: {
[primaryKey]: results.insertId
},
}
}
return json
}
async query(json: QueryJson) {
const operation = this._operation(json)
this.client.connect()
@ -254,7 +271,7 @@ module MySQLModule {
const results = await internalQuery(this.client, input, false)
// same as delete, manage returning
if (operation === Operation.CREATE || operation === Operation.UPDATE) {
row = this.getReturningRow(json)
row = this.getReturningRow(this.checkLookupKeys(results, json))
}
this.client.end()
if (operation !== Operation.READ) {

View File

@ -57,7 +57,7 @@ describe("SQL query builder", () => {
const query = sql._query(generateReadJson())
expect(query).toEqual({
bindings: [limit],
sql: `select * from "${TABLE_NAME}" limit $1`
sql: `select * from (select * from "${TABLE_NAME}" limit $1) as "${TABLE_NAME}"`
})
})
@ -68,7 +68,7 @@ describe("SQL query builder", () => {
}))
expect(query).toEqual({
bindings: [limit],
sql: `select "${TABLE_NAME}"."name" as "${nameProp}", "${TABLE_NAME}"."age" as "${ageProp}" from "${TABLE_NAME}" limit $1`
sql: `select "${TABLE_NAME}"."name" as "${nameProp}", "${TABLE_NAME}"."age" as "${ageProp}" from (select * from "${TABLE_NAME}" limit $1) as "${TABLE_NAME}"`
})
})
@ -82,7 +82,7 @@ describe("SQL query builder", () => {
}))
expect(query).toEqual({
bindings: ["John%", limit],
sql: `select * from "${TABLE_NAME}" where "${TABLE_NAME}"."name" ilike $1 limit $2`
sql: `select * from (select * from "${TABLE_NAME}" where "${TABLE_NAME}"."name" ilike $1 limit $2) as "${TABLE_NAME}"`
})
})
@ -99,7 +99,7 @@ describe("SQL query builder", () => {
}))
expect(query).toEqual({
bindings: [2, 10, limit],
sql: `select * from "${TABLE_NAME}" where "${TABLE_NAME}"."age" between $1 and $2 limit $3`
sql: `select * from (select * from "${TABLE_NAME}" where "${TABLE_NAME}"."age" between $1 and $2 limit $3) as "${TABLE_NAME}"`
})
})
@ -115,7 +115,7 @@ describe("SQL query builder", () => {
}))
expect(query).toEqual({
bindings: [10, "John", limit],
sql: `select * from "${TABLE_NAME}" where ("${TABLE_NAME}"."age" = $1) or ("${TABLE_NAME}"."name" = $2) limit $3`
sql: `select * from (select * from "${TABLE_NAME}" where ("${TABLE_NAME}"."age" = $1) or ("${TABLE_NAME}"."name" = $2) limit $3) as "${TABLE_NAME}"`
})
})
@ -160,7 +160,7 @@ describe("SQL query builder", () => {
const query = new Sql("mssql", 10)._query(generateReadJson())
expect(query).toEqual({
bindings: [10],
sql: `select top (@p0) * from [${TABLE_NAME}]`
sql: `select * from (select top (@p0) * from [${TABLE_NAME}]) as [${TABLE_NAME}]`
})
})
@ -168,7 +168,7 @@ describe("SQL query builder", () => {
const query = new Sql("mysql", 10)._query(generateReadJson())
expect(query).toEqual({
bindings: [10],
sql: `select * from \`${TABLE_NAME}\` limit ?`
sql: `select * from (select * from \`${TABLE_NAME}\` limit ?) as \`${TABLE_NAME}\``
})
})
})

View File

@ -1,6 +1,6 @@
{
"name": "@budibase/string-templates",
"version": "0.9.140-alpha.8",
"version": "0.9.140-alpha.11",
"description": "Handlebars wrapper for Budibase templating.",
"main": "src/index.cjs",
"module": "dist/bundle.mjs",

View File

@ -1,7 +1,7 @@
{
"name": "@budibase/worker",
"email": "hi@budibase.com",
"version": "0.9.140-alpha.8",
"version": "0.9.140-alpha.11",
"description": "Budibase background service",
"main": "src/index.js",
"repository": {
@ -25,8 +25,8 @@
"author": "Budibase",
"license": "AGPL-3.0-or-later",
"dependencies": {
"@budibase/auth": "^0.9.140-alpha.8",
"@budibase/string-templates": "^0.9.140-alpha.8",
"@budibase/auth": "^0.9.140-alpha.11",
"@budibase/string-templates": "^0.9.140-alpha.11",
"@koa/router": "^8.0.0",
"@techpass/passport-openidconnect": "^0.3.0",
"aws-sdk": "^2.811.0",