Merge branch 'develop' of github.com:Budibase/budibase into data-ui-pagination

lerna.json

@@ -1,5 +1,5 @@
 {
-  "version": "0.9.140-alpha.5",
+  "version": "0.9.140-alpha.11",
   "npmClient": "yarn",
   "packages": [
     "packages/*"

packages/auth/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@budibase/auth",
-  "version": "0.9.140-alpha.5",
+  "version": "0.9.140-alpha.11",
   "description": "Authentication middlewares for budibase builder and apps",
   "main": "src/index.js",
   "author": "Budibase",

packages/bbui/package.json

@@ -1,7 +1,7 @@
 {
   "name": "@budibase/bbui",
   "description": "A UI solution used in the different Budibase projects.",
-  "version": "0.9.140-alpha.5",
+  "version": "0.9.140-alpha.11",
   "license": "AGPL-3.0",
   "svelte": "src/index.js",
   "module": "dist/bbui.es.js",

packages/builder/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@budibase/builder",
-  "version": "0.9.140-alpha.5",
+  "version": "0.9.140-alpha.11",
   "license": "AGPL-3.0",
   "private": true,
   "scripts": {
@@ -65,10 +65,10 @@
     }
   },
   "dependencies": {
-    "@budibase/bbui": "^0.9.140-alpha.5",
+    "@budibase/bbui": "^0.9.140-alpha.11",
-    "@budibase/client": "^0.9.140-alpha.5",
+    "@budibase/client": "^0.9.140-alpha.11",
     "@budibase/colorpicker": "1.1.2",
-    "@budibase/string-templates": "^0.9.140-alpha.5",
+    "@budibase/string-templates": "^0.9.140-alpha.11",
     "@sentry/browser": "5.19.1",
     "@spectrum-css/page": "^3.0.1",
     "@spectrum-css/vars": "^3.0.1",

packages/builder/src/builderStore/dataBinding.js

@@ -443,10 +443,9 @@ function bindingReplacement(bindableProperties, textWithBindings, convertTo) {
     for (let from of convertFromProps) {
       if (shouldReplaceBinding(newBoundValue, from, convertTo)) {
         const binding = bindableProperties.find(el => el[convertFrom] === from)
-        newBoundValue = newBoundValue.replace(
+        while (newBoundValue.includes(from)) {
-          new RegExp(from, "gi"),
+          newBoundValue = newBoundValue.replace(from, binding[convertTo])
-          binding[convertTo]
+        }
-        )
       }
     }
     result = result.replace(boundValue, newBoundValue)

packages/builder/src/components/backend/DatasourceNavigator/DatasourceNavigator.svelte

@@ -1,8 +1,9 @@
 <script>
   import { onMount } from "svelte"
+  import { get } from "svelte/store"
   import { goto } from "@roxi/routify"
   import { BUDIBASE_INTERNAL_DB } from "constants"
-  import { database, datasources, queries } from "stores/backend"
+  import { database, datasources, queries, tables } from "stores/backend"
   import EditDatasourcePopover from "./popovers/EditDatasourcePopover.svelte"
   import EditQueryPopover from "./popovers/EditQueryPopover.svelte"
   import NavItem from "components/common/NavItem.svelte"
@@ -10,6 +11,13 @@
   import ICONS from "./icons"
 
   let openDataSources = []
+  $: enrichedDataSources = $datasources.list.map(datasource => ({
+    ...datasource,
+    open:
+      openDataSources.includes(datasource._id) ||
+      containsActiveTable(datasource),
+    selected: $datasources.selected === datasource._id,
+  }))
 
   function selectDatasource(datasource) {
     toggleNode(datasource)
@@ -35,16 +43,28 @@
     datasources.fetch()
     queries.fetch()
   })
+
+  const containsActiveTable = datasource => {
+    const activeTableId = get(tables).selected?._id
+    if (!datasource.entities) {
+      return false
+    }
+    let tableOptions = datasource.entities
+    if (!Array.isArray(tableOptions)) {
+      tableOptions = Object.values(tableOptions)
+    }
+    return tableOptions.find(x => x._id === activeTableId) != null
+  }
 </script>
 
 {#if $database?._id}
   <div class="hierarchy-items-container">
-    {#each $datasources.list as datasource, idx}
+    {#each enrichedDataSources as datasource, idx}
       <NavItem
         border={idx > 0}
         text={datasource.name}
-        opened={openDataSources.includes(datasource._id)}
+        opened={datasource.open}
-        selected={$datasources.selected === datasource._id}
+        selected={datasource.selected}
         withArrow={true}
         on:click={() => selectDatasource(datasource)}
         on:iconClick={() => toggleNode(datasource)}
@@ -61,7 +81,7 @@
         {/if}
       </NavItem>
 
-      {#if openDataSources.includes(datasource._id)}
+      {#if datasource.open}
         <TableNavigator sourceId={datasource._id} />
         {#each $queries.list.filter(query => query.datasourceId === datasource._id) as query}
           <NavItem

packages/builder/src/components/design/PropertiesPanel/PropertyControls/EventsEditor/actions/RefreshDataProvider.svelte

@@ -0,0 +1,35 @@
+<script>
+  import { Select, Label } from "@budibase/bbui"
+  import { currentAsset, store } from "builderStore"
+  import { getActionProviderComponents } from "builderStore/dataBinding"
+
+  export let parameters
+
+  $: actionProviders = getActionProviderComponents(
+    $currentAsset,
+    $store.selectedComponentId,
+    "RefreshDataProvider"
+  )
+</script>
+
+<div class="root">
+  <Label small>Data Provider</Label>
+  <Select
+    bind:value={parameters.componentId}
+    options={actionProviders}
+    getOptionLabel={x => x._instanceName}
+    getOptionValue={x => x._id}
+  />
+</div>
+
+<style>
+  .root {
+    display: grid;
+    column-gap: var(--spacing-l);
+    row-gap: var(--spacing-s);
+    grid-template-columns: 70px 1fr;
+    align-items: center;
+    max-width: 400px;
+    margin: 0 auto;
+  }
+</style>

packages/builder/src/components/design/PropertiesPanel/PropertyControls/EventsEditor/actions/index.js

@@ -12,6 +12,7 @@ import ClearForm from "./ClearForm.svelte"
 import CloseScreenModal from "./CloseScreenModal.svelte"
 import ChangeFormStep from "./ChangeFormStep.svelte"
 import UpdateStateStep from "./UpdateState.svelte"
+import RefreshDataProvider from "./RefreshDataProvider.svelte"
 
 // Defines which actions are available to configure in the front end.
 // Unfortunately the "name" property is used as the identifier so please don't
@@ -62,6 +63,10 @@ export const getAvailableActions = () => {
       name: "Change Form Step",
       component: ChangeFormStep,
     },
+    {
+      name: "Refresh Data Provider",
+      component: RefreshDataProvider,
+    },
   ]
 
   if (get(store).clientFeatures?.state) {

packages/builder/src/pages/builder/app/[application]/data/datasource/bb_internal/_layout.svelte

@@ -0,0 +1,7 @@
+<script>
+  import { datasources } from "stores/backend"
+
+  datasources.select("bb_internal")
+</script>
+
+<slot />

packages/builder/src/stores/backend/datasources.js

@@ -1,4 +1,4 @@
-import { writable } from "svelte/store"
+import { writable, get } from "svelte/store"
 import { queries, tables, views } from "./"
 import api from "../../builderStore/api"
 
@@ -8,7 +8,8 @@ export const INITIAL_DATASOURCE_VALUES = {
 }
 
 export function createDatasourcesStore() {
-  const { subscribe, update, set } = writable(INITIAL_DATASOURCE_VALUES)
+  const store = writable(INITIAL_DATASOURCE_VALUES)
+  const { subscribe, update, set } = store
 
   return {
     subscribe,
@@ -21,7 +22,15 @@ export function createDatasourcesStore() {
     fetch: async () => {
       const response = await api.get(`/api/datasources`)
       const json = await response.json()
-      update(state => ({ ...state, list: json, selected: null }))
+
+      // Clear selected if it no longer exists, otherwise keep it
+      const selected = get(store).selected
+      let nextSelected = null
+      if (selected && json.find(source => source._id === selected)) {
+        nextSelected = selected
+      }
+
+      update(state => ({ ...state, list: json, selected: nextSelected }))
       return json
     },
     select: async datasourceId => {

packages/cli/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@budibase/cli",
-  "version": "0.9.140-alpha.5",
+  "version": "0.9.140-alpha.11",
   "description": "Budibase CLI, for developers, self hosting and migrations.",
   "main": "src/index.js",
   "bin": {

packages/client/manifest.json

@@ -2389,6 +2389,7 @@
     "icon": "Data",
     "illegalChildren": ["section"],
     "hasChildren": true,
+    "actions": ["RefreshDatasource"],
     "settings": [
       {
         "type": "dataSource",

packages/client/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@budibase/client",
-  "version": "0.9.140-alpha.5",
+  "version": "0.9.140-alpha.11",
   "license": "MPL-2.0",
   "module": "dist/budibase-client.js",
   "main": "dist/budibase-client.js",
@@ -19,9 +19,9 @@
     "dev:builder": "rollup -cw"
   },
   "dependencies": {
-    "@budibase/bbui": "^0.9.140-alpha.5",
+    "@budibase/bbui": "^0.9.140-alpha.11",
     "@budibase/standard-components": "^0.9.139",
-    "@budibase/string-templates": "^0.9.140-alpha.5",
+    "@budibase/string-templates": "^0.9.140-alpha.11",
     "regexparam": "^1.3.0",
     "shortid": "^2.2.15",
     "svelte-spa-router": "^3.0.5"

packages/client/src/utils/buttonActions.js

@@ -88,7 +88,7 @@ const validateFormHandler = async (action, context) => {
   )
 }
 
-const refreshDatasourceHandler = async (action, context) => {
+const refreshDataProviderHandler = async (action, context) => {
   return await executeActionHandler(
     context,
     action.parameters.componentId,
@@ -139,7 +139,7 @@ const handlerMap = {
   ["Execute Query"]: queryExecutionHandler,
   ["Trigger Automation"]: triggerAutomationHandler,
   ["Validate Form"]: validateFormHandler,
-  ["Refresh Datasource"]: refreshDatasourceHandler,
+  ["Refresh Data Provider"]: refreshDataProviderHandler,
   ["Log Out"]: logoutHandler,
   ["Clear Form"]: clearFormHandler,
   ["Close Screen Modal"]: closeScreenModalHandler,

packages/server/package.json

@@ -1,7 +1,7 @@
 {
   "name": "@budibase/server",
   "email": "hi@budibase.com",
-  "version": "0.9.140-alpha.5",
+  "version": "0.9.140-alpha.11",
   "description": "Budibase Web Server",
   "main": "src/index.js",
   "repository": {
@@ -13,7 +13,7 @@
     "postbuild": "copyfiles -u 1 src/**/*.svelte dist/ && copyfiles -u 1 src/**/*.hbs dist/ && copyfiles -u 1 src/**/*.json dist/",
     "test": "jest --coverage --maxWorkers=2",
     "test:watch": "jest --watch",
-    "predocker": "copyfiles -f ../client/dist/budibase-client.js ../standard-components/manifest.json client",
+    "predocker": "copyfiles -f ../client/dist/budibase-client.js ../client/manifest.json client",
     "build:docker": "yarn run predocker && docker build . -t app-service",
     "run:docker": "node dist/index.js",
     "dev:stack:up": "node scripts/dev/manage.js up",
@@ -62,9 +62,9 @@
   "author": "Budibase",
   "license": "AGPL-3.0-or-later",
   "dependencies": {
-    "@budibase/auth": "^0.9.140-alpha.5",
+    "@budibase/auth": "^0.9.140-alpha.11",
-    "@budibase/client": "^0.9.140-alpha.5",
+    "@budibase/client": "^0.9.140-alpha.11",
-    "@budibase/string-templates": "^0.9.140-alpha.5",
+    "@budibase/string-templates": "^0.9.140-alpha.11",
     "@elastic/elasticsearch": "7.10.0",
     "@koa/router": "8.0.0",
     "@sendgrid/mail": "7.1.1",

packages/server/scripts/integrations/postgres/docker-compose.yml

@@ -15,7 +15,7 @@ services:
       - ./init.sql:/docker-entrypoint-initdb.d/init.sql
 
   pgadmin:
-    container_name: pgadmin
+    container_name: pgadmin-pg
     image: dpage/pgadmin4
     restart: always
     environment:

packages/server/src/api/controllers/application.js

@@ -230,7 +230,12 @@ exports.create = async function (ctx) {
   const response = await db.put(newApplication, { force: true })
   newApplication._rev = response.rev
 
+  // Only create the default home screens and layout if we aren't importing
+  // an app
+  if (useTemplate !== "true") {
     await createEmptyAppPackage(ctx, newApplication)
+  }
+
   /* istanbul ignore next */
   if (!env.isTest()) {
     await createApp(appId)

packages/server/src/api/controllers/row/ExternalRequest.ts

@@ -544,6 +544,9 @@ module External {
         extra: {
           idFilter: buildFilters(id || generateIdForRow(row, table), {}, table),
         },
+        meta: {
+          table,
+        }
       }
       // can't really use response right now
       const response = await makeExternalQuery(appId, json)

packages/server/src/api/routes/tests/datasource.spec.js

@@ -94,7 +94,8 @@ describe("/datasources", () => {
         .expect(200)
       // this is mock data, can't test it
       expect(res.body).toBeDefined()
-      expect(pg.queryMock).toHaveBeenCalledWith(`select "users"."name" as "users.name", "users"."age" as "users.age" from "users" where "users"."name" ilike $1 limit $2`, ["John%", 5000])
+      const expSql = `select "users"."name" as "users.name", "users"."age" as "users.age" from (select * from "users" where "users"."name" ilike $1 limit $2) as "users"`
+      expect(pg.queryMock).toHaveBeenCalledWith(expSql, ["John%", 5000])
     })
   })
 

packages/server/src/definitions/datasource.ts

@@ -1,3 +1,5 @@
+import {Table} from "./common";
+
 export enum Operation {
   CREATE = "CREATE",
   READ = "READ",
@@ -136,6 +138,9 @@ export interface QueryJson {
   sort?: SortJson
   paginate?: PaginationJson
   body?: object
+  meta?: {
+    table?: Table,
+  }
   extra?: {
     idFilter?: SearchFilters
   }

packages/server/src/integrations/base/sql.ts

@@ -1,7 +1,5 @@
 import { Knex, knex } from "knex"
 const BASE_LIMIT = 5000
-// if requesting a single row then need to up the limit for the sake of joins
-const SINGLE_ROW_LIMIT = 100
 import {
   QueryJson,
   SearchFilters,
@@ -146,46 +144,48 @@ function buildCreate(
 function buildRead(knex: Knex, json: QueryJson, limit: number): KnexQuery {
   let { endpoint, resource, filters, sort, paginate, relationships } = json
   const tableName = endpoint.entityId
-  let query: KnexQuery = knex(tableName)
   // select all if not specified
   if (!resource) {
     resource = { fields: [] }
   }
+  let selectStatement: string|string[] = "*"
   // handle select
   if (resource.fields && resource.fields.length > 0) {
     // select the resources as the format "table.columnName" - this is what is provided
     // by the resource builder further up
-    query = query.select(resource.fields.map(field => `${field} as ${field}`))
+    selectStatement = resource.fields.map(field => `${field} as ${field}`)
-  } else {
+  }
-    query = query.select("*")
+  let foundLimit = limit || BASE_LIMIT
+  // handle pagination
+  let foundOffset: number | null = null
+  if (paginate && paginate.page && paginate.limit) {
+    // @ts-ignore
+    const page = paginate.page <= 1 ? 0 : paginate.page - 1
+    const offset = page * paginate.limit
+    foundLimit = paginate.limit
+    foundOffset = offset
+  } else if (paginate && paginate.limit) {
+    foundLimit = paginate.limit
+  }
+  // start building the query
+  let query: KnexQuery = knex(tableName).limit(foundLimit)
+  if (foundOffset) {
+    query = query.offset(foundOffset)
   }
-  // handle where
-  query = addFilters(tableName, query, filters)
-  // handle join
-  query = addRelationships(query, tableName, relationships)
-  // handle sorting
   if (sort) {
     for (let [key, value] of Object.entries(sort)) {
       const direction = value === SortDirection.ASCENDING ? "asc" : "desc"
       query = query.orderBy(key, direction)
     }
   }
-  let foundLimit = limit || BASE_LIMIT
+  query = addFilters(tableName, query, filters)
-  // handle pagination
-  if (paginate && paginate.page && paginate.limit) {
   // @ts-ignore
-    const page = paginate.page <= 1 ? 0 : paginate.page - 1
+  let preQuery: KnexQuery = knex({
-    const offset = page * paginate.limit
+    // @ts-ignore
-    foundLimit = paginate.limit
+    [tableName]: query,
-    query = query.offset(offset)
+  }).select(selectStatement)
-  } else if (paginate && paginate.limit) {
+  // handle joins
-    foundLimit = paginate.limit
+  return addRelationships(preQuery, tableName, relationships)
-  }
-  if (foundLimit === 1) {
-    foundLimit = SINGLE_ROW_LIMIT
-  }
-  query = query.limit(foundLimit)
-  return query
 }
 
 function buildUpdate(

packages/server/src/integrations/mysql.ts

@@ -104,7 +104,7 @@ module MySQLModule {
     client: any,
     query: SqlQuery,
     connect: boolean = true
-  ): Promise<any[]> {
+  ): Promise<any[]|any> {
     // Node MySQL is callback based, so we must wrap our call in a promise
     return new Promise((resolve, reject) => {
       if (connect) {
@@ -238,6 +238,23 @@ module MySQLModule {
       return internalQuery(this.client, input, false)
     }
 
+    // when creating if an ID has been inserted need to make sure
+    // the id filter is enriched with it before trying to retrieve the row
+    checkLookupKeys(results: any, json: QueryJson) {
+      if (!results?.insertId || !json.meta?.table || !json.meta.table.primary) {
+        return json
+      }
+      const primaryKey = json.meta.table.primary?.[0]
+      json.extra = {
+        idFilter: {
+          equal: {
+            [primaryKey]: results.insertId
+          },
+        }
+      }
+      return json
+    }
+
     async query(json: QueryJson) {
       const operation = this._operation(json)
       this.client.connect()
@@ -250,7 +267,7 @@ module MySQLModule {
       const results = await internalQuery(this.client, input, false)
       // same as delete, manage returning
       if (operation === Operation.CREATE || operation === Operation.UPDATE) {
-        row = this.getReturningRow(json)
+        row = this.getReturningRow(this.checkLookupKeys(results, json))
       }
       this.client.end()
       if (operation !== Operation.READ) {

packages/server/src/integrations/postgres.ts

@@ -84,10 +84,13 @@ module PostgresModule {
     bigint: FieldTypes.NUMBER,
     decimal: FieldTypes.NUMBER,
     smallint: FieldTypes.NUMBER,
+    real: FieldTypes.NUMBER,
+    "double precision": FieldTypes.NUMBER,
     timestamp: FieldTypes.DATETIME,
     time: FieldTypes.DATETIME,
     boolean: FieldTypes.BOOLEAN,
     json: FieldTypes.JSON,
+    date: FieldTypes.DATETIME,
   }
 
   async function internalQuery(client: any, query: SqlQuery) {

packages/server/src/integrations/tests/sql.spec.js

@@ -57,7 +57,7 @@ describe("SQL query builder", () => {
     const query = sql._query(generateReadJson())
     expect(query).toEqual({
       bindings: [limit],
-      sql: `select * from "${TABLE_NAME}" limit $1`
+      sql: `select * from (select * from "${TABLE_NAME}" limit $1) as "${TABLE_NAME}"`
     })
   })
 
@@ -68,7 +68,7 @@ describe("SQL query builder", () => {
     }))
     expect(query).toEqual({
       bindings: [limit],
-      sql: `select "${TABLE_NAME}"."name" as "${nameProp}", "${TABLE_NAME}"."age" as "${ageProp}" from "${TABLE_NAME}" limit $1`
+      sql: `select "${TABLE_NAME}"."name" as "${nameProp}", "${TABLE_NAME}"."age" as "${ageProp}" from (select * from "${TABLE_NAME}" limit $1) as "${TABLE_NAME}"`
     })
   })
 
@@ -82,7 +82,7 @@ describe("SQL query builder", () => {
     }))
     expect(query).toEqual({
       bindings: ["John%", limit],
-      sql: `select * from "${TABLE_NAME}" where "${TABLE_NAME}"."name" ilike $1 limit $2`
+      sql: `select * from (select * from "${TABLE_NAME}" where "${TABLE_NAME}"."name" ilike $1 limit $2) as "${TABLE_NAME}"`
     })
   })
 
@@ -99,7 +99,7 @@ describe("SQL query builder", () => {
     }))
     expect(query).toEqual({
       bindings: [2, 10, limit],
-      sql: `select * from "${TABLE_NAME}" where "${TABLE_NAME}"."age" between $1 and $2 limit $3`
+      sql: `select * from (select * from "${TABLE_NAME}" where "${TABLE_NAME}"."age" between $1 and $2 limit $3) as "${TABLE_NAME}"`
     })
   })
 
@@ -115,7 +115,7 @@ describe("SQL query builder", () => {
     }))
     expect(query).toEqual({
       bindings: [10, "John", limit],
-      sql: `select * from "${TABLE_NAME}" where ("${TABLE_NAME}"."age" = $1) or ("${TABLE_NAME}"."name" = $2) limit $3`
+      sql: `select * from (select * from "${TABLE_NAME}" where ("${TABLE_NAME}"."age" = $1) or ("${TABLE_NAME}"."name" = $2) limit $3) as "${TABLE_NAME}"`
     })
   })
 
@@ -160,7 +160,7 @@ describe("SQL query builder", () => {
     const query = new Sql("mssql", 10)._query(generateReadJson())
     expect(query).toEqual({
       bindings: [10],
-      sql: `select top (@p0) * from [${TABLE_NAME}]`
+      sql: `select * from (select top (@p0) * from [${TABLE_NAME}]) as [${TABLE_NAME}]`
     })
   })
 
@@ -168,7 +168,7 @@ describe("SQL query builder", () => {
     const query = new Sql("mysql", 10)._query(generateReadJson())
     expect(query).toEqual({
       bindings: [10],
-      sql: `select * from \`${TABLE_NAME}\` limit ?`
+      sql: `select * from (select * from \`${TABLE_NAME}\` limit ?) as \`${TABLE_NAME}\``
     })
   })
 })

packages/string-templates/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@budibase/string-templates",
-  "version": "0.9.140-alpha.5",
+  "version": "0.9.140-alpha.11",
   "description": "Handlebars wrapper for Budibase templating.",
   "main": "src/index.cjs",
   "module": "dist/bundle.mjs",

packages/worker/package.json

@@ -1,7 +1,7 @@
 {
   "name": "@budibase/worker",
   "email": "hi@budibase.com",
-  "version": "0.9.140-alpha.5",
+  "version": "0.9.140-alpha.11",
   "description": "Budibase background service",
   "main": "src/index.js",
   "repository": {
@@ -25,8 +25,8 @@
   "author": "Budibase",
   "license": "AGPL-3.0-or-later",
   "dependencies": {
-    "@budibase/auth": "^0.9.140-alpha.5",
+    "@budibase/auth": "^0.9.140-alpha.11",
-    "@budibase/string-templates": "^0.9.140-alpha.5",
+    "@budibase/string-templates": "^0.9.140-alpha.11",
     "@koa/router": "^8.0.0",
     "@techpass/passport-openidconnect": "^0.3.0",
     "aws-sdk": "^2.811.0",