Merge branch 'develop' of github.com:Budibase/budibase into ak-fixes

lerna.json

@@ -1,5 +1,5 @@
 {
-  "version": "0.9.140-alpha.5",
+  "version": "0.9.140-alpha.10",
   "npmClient": "yarn",
   "packages": [
     "packages/*"

packages/auth/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@budibase/auth",
-  "version": "0.9.140-alpha.5",
+  "version": "0.9.140-alpha.10",
   "description": "Authentication middlewares for budibase builder and apps",
   "main": "src/index.js",
   "author": "Budibase",

packages/bbui/package.json

@@ -1,7 +1,7 @@
 {
   "name": "@budibase/bbui",
   "description": "A UI solution used in the different Budibase projects.",
-  "version": "0.9.140-alpha.5",
+  "version": "0.9.140-alpha.10",
   "license": "AGPL-3.0",
   "svelte": "src/index.js",
   "module": "dist/bbui.es.js",

packages/builder/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@budibase/builder",
-  "version": "0.9.140-alpha.5",
+  "version": "0.9.140-alpha.10",
   "license": "AGPL-3.0",
   "private": true,
   "scripts": {
@@ -65,10 +65,10 @@
     }
   },
   "dependencies": {
-    "@budibase/bbui": "^0.9.140-alpha.5",
+    "@budibase/bbui": "^0.9.140-alpha.10",
-    "@budibase/client": "^0.9.140-alpha.5",
+    "@budibase/client": "^0.9.140-alpha.10",
     "@budibase/colorpicker": "1.1.2",
-    "@budibase/string-templates": "^0.9.140-alpha.5",
+    "@budibase/string-templates": "^0.9.140-alpha.10",
     "@sentry/browser": "5.19.1",
     "@spectrum-css/page": "^3.0.1",
     "@spectrum-css/vars": "^3.0.1",

packages/builder/src/builderStore/store/frontend.js

@@ -215,6 +215,13 @@ export const getFrontendStore = () => {
             if (screenToDelete._id === state.selectedScreenId) {
               state.selectedScreenId = null
             }
+            //remove the link for this screen
+            screenDeletePromises.push(
+              store.actions.components.links.delete(
+                screenToDelete.routing.route,
+                screenToDelete.props._instanceName
+              )
+            )
           }
           return state
         })
@@ -646,6 +653,36 @@ export const getFrontendStore = () => {
           // Save layout
           await store.actions.layouts.save(layout)
         },
+        delete: async (url, title) => {
+          const layout = get(mainLayout)
+          if (!layout) {
+            return
+          }
+
+          // Add link setting to main layout
+          if (layout.props._component.endsWith("layout")) {
+            // If using a new SDK, add to the layout component settings
+            layout.props.links = layout.props.links.filter(
+              link => !(link.text === title && link.url === url)
+            )
+          } else {
+            // If using an old SDK, add to the navigation component
+            // TODO: remove this when we can assume everyone has updated
+            const nav = findComponentType(
+              layout.props,
+              "@budibase/standard-components/navigation"
+            )
+            if (!nav) {
+              return
+            }
+
+            nav._children = nav._children.filter(
+              child => !(child.url === url && child.text === title)
+            )
+          }
+          // Save layout
+          await store.actions.layouts.save(layout)
+        },
       },
     },
   }

packages/builder/src/components/backend/DatasourceNavigator/DatasourceNavigator.svelte

@@ -83,20 +83,19 @@
 
       {#if datasource.open}
         <TableNavigator sourceId={datasource._id} />
+        {#each $queries.list.filter(query => query.datasourceId === datasource._id) as query}
+          <NavItem
+            indentLevel={1}
+            icon="SQLQuery"
+            text={query.name}
+            opened={$queries.selected === query._id}
+            selected={$queries.selected === query._id}
+            on:click={() => onClickQuery(query)}
+          >
+            <EditQueryPopover {query} />
+          </NavItem>
+        {/each}
       {/if}
-
-      {#each $queries.list.filter(query => query.datasourceId === datasource._id) as query}
-        <NavItem
-          indentLevel={1}
-          icon="SQLQuery"
-          text={query.name}
-          opened={$queries.selected === query._id}
-          selected={$queries.selected === query._id}
-          on:click={() => onClickQuery(query)}
-        >
-          <EditQueryPopover {query} />
-        </NavItem>
-      {/each}
     {/each}
   </div>
 {/if}

packages/builder/src/components/design/PropertiesPanel/PropertyControls/EventsEditor/actions/RefreshDataProvider.svelte

@@ -0,0 +1,35 @@
+<script>
+  import { Select, Label } from "@budibase/bbui"
+  import { currentAsset, store } from "builderStore"
+  import { getActionProviderComponents } from "builderStore/dataBinding"
+
+  export let parameters
+
+  $: actionProviders = getActionProviderComponents(
+    $currentAsset,
+    $store.selectedComponentId,
+    "RefreshDataProvider"
+  )
+</script>
+
+<div class="root">
+  <Label small>Data Provider</Label>
+  <Select
+    bind:value={parameters.componentId}
+    options={actionProviders}
+    getOptionLabel={x => x._instanceName}
+    getOptionValue={x => x._id}
+  />
+</div>
+
+<style>
+  .root {
+    display: grid;
+    column-gap: var(--spacing-l);
+    row-gap: var(--spacing-s);
+    grid-template-columns: 70px 1fr;
+    align-items: center;
+    max-width: 400px;
+    margin: 0 auto;
+  }
+</style>

packages/builder/src/components/design/PropertiesPanel/PropertyControls/EventsEditor/actions/index.js

@@ -12,6 +12,7 @@ import ClearForm from "./ClearForm.svelte"
 import CloseScreenModal from "./CloseScreenModal.svelte"
 import ChangeFormStep from "./ChangeFormStep.svelte"
 import UpdateStateStep from "./UpdateState.svelte"
+import RefreshDataProvider from "./RefreshDataProvider.svelte"
 
 // Defines which actions are available to configure in the front end.
 // Unfortunately the "name" property is used as the identifier so please don't
@@ -62,6 +63,10 @@ export const getAvailableActions = () => {
       name: "Change Form Step",
       component: ChangeFormStep,
     },
+    {
+      name: "Refresh Data Provider",
+      component: RefreshDataProvider,
+    },
   ]
 
   if (get(store).clientFeatures?.state) {

packages/cli/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@budibase/cli",
-  "version": "0.9.140-alpha.5",
+  "version": "0.9.140-alpha.10",
   "description": "Budibase CLI, for developers, self hosting and migrations.",
   "main": "src/index.js",
   "bin": {

packages/client/manifest.json

@@ -2389,6 +2389,7 @@
     "icon": "Data",
     "illegalChildren": ["section"],
     "hasChildren": true,
+    "actions": ["RefreshDatasource"],
     "settings": [
       {
         "type": "dataSource",

packages/client/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@budibase/client",
-  "version": "0.9.140-alpha.5",
+  "version": "0.9.140-alpha.10",
   "license": "MPL-2.0",
   "module": "dist/budibase-client.js",
   "main": "dist/budibase-client.js",
@@ -19,9 +19,9 @@
     "dev:builder": "rollup -cw"
   },
   "dependencies": {
-    "@budibase/bbui": "^0.9.140-alpha.5",
+    "@budibase/bbui": "^0.9.140-alpha.10",
     "@budibase/standard-components": "^0.9.139",
-    "@budibase/string-templates": "^0.9.140-alpha.5",
+    "@budibase/string-templates": "^0.9.140-alpha.10",
     "regexparam": "^1.3.0",
     "shortid": "^2.2.15",
     "svelte-spa-router": "^3.0.5"

packages/client/src/utils/buttonActions.js

@@ -88,7 +88,7 @@ const validateFormHandler = async (action, context) => {
   )
 }
 
-const refreshDatasourceHandler = async (action, context) => {
+const refreshDataProviderHandler = async (action, context) => {
   return await executeActionHandler(
     context,
     action.parameters.componentId,
@@ -139,7 +139,7 @@ const handlerMap = {
   ["Execute Query"]: queryExecutionHandler,
   ["Trigger Automation"]: triggerAutomationHandler,
   ["Validate Form"]: validateFormHandler,
-  ["Refresh Datasource"]: refreshDatasourceHandler,
+  ["Refresh Data Provider"]: refreshDataProviderHandler,
   ["Log Out"]: logoutHandler,
   ["Clear Form"]: clearFormHandler,
   ["Close Screen Modal"]: closeScreenModalHandler,

packages/server/package.json

@@ -1,7 +1,7 @@
 {
   "name": "@budibase/server",
   "email": "hi@budibase.com",
-  "version": "0.9.140-alpha.5",
+  "version": "0.9.140-alpha.10",
   "description": "Budibase Web Server",
   "main": "src/index.js",
   "repository": {
@@ -13,7 +13,7 @@
     "postbuild": "copyfiles -u 1 src/**/*.svelte dist/ && copyfiles -u 1 src/**/*.hbs dist/ && copyfiles -u 1 src/**/*.json dist/",
     "test": "jest --coverage --maxWorkers=2",
     "test:watch": "jest --watch",
-    "predocker": "copyfiles -f ../client/dist/budibase-client.js ../standard-components/manifest.json client",
+    "predocker": "copyfiles -f ../client/dist/budibase-client.js ../client/manifest.json client",
     "build:docker": "yarn run predocker && docker build . -t app-service",
     "run:docker": "node dist/index.js",
     "dev:stack:up": "node scripts/dev/manage.js up",
@@ -62,9 +62,9 @@
   "author": "Budibase",
   "license": "AGPL-3.0-or-later",
   "dependencies": {
-    "@budibase/auth": "^0.9.140-alpha.5",
+    "@budibase/auth": "^0.9.140-alpha.10",
-    "@budibase/client": "^0.9.140-alpha.5",
+    "@budibase/client": "^0.9.140-alpha.10",
-    "@budibase/string-templates": "^0.9.140-alpha.5",
+    "@budibase/string-templates": "^0.9.140-alpha.10",
     "@elastic/elasticsearch": "7.10.0",
     "@koa/router": "8.0.0",
     "@sendgrid/mail": "7.1.1",

packages/server/scripts/integrations/postgres/docker-compose.yml

@@ -15,7 +15,7 @@ services:
       - ./init.sql:/docker-entrypoint-initdb.d/init.sql
 
   pgadmin:
-    container_name: pgadmin
+    container_name: pgadmin-pg
     image: dpage/pgadmin4
     restart: always
     environment:

packages/server/src/api/controllers/row/ExternalRequest.ts

@@ -544,6 +544,9 @@ module External {
         extra: {
           idFilter: buildFilters(id || generateIdForRow(row, table), {}, table),
         },
+        meta: {
+          table,
+        }
       }
       // can't really use response right now
       const response = await makeExternalQuery(appId, json)

packages/server/src/api/controllers/row/utils.js

@@ -5,6 +5,7 @@ const { InternalTables } = require("../../../db/utils")
 const userController = require("../user")
 const { FieldTypes } = require("../../../constants")
 const { integrations } = require("../../../integrations")
+const { processStringSync } = require("@budibase/string-templates")
 
 validateJs.extend(validateJs.validators.datetime, {
   parse: function (value) {
@@ -73,6 +74,11 @@ exports.validate = async ({ appId, tableId, row, table }) => {
           errors[fieldName] = "Field not in list"
         }
       })
+    } else if (table.schema[fieldName].type === FieldTypes.FORMULA) {
+      res = validateJs.single(
+        processStringSync(table.schema[fieldName].formula, row),
+        constraints
+      )
     } else {
       res = validateJs.single(row[fieldName], constraints)
     }

packages/server/src/api/routes/tests/datasource.spec.js

@@ -94,7 +94,8 @@ describe("/datasources", () => {
         .expect(200)
       // this is mock data, can't test it
       expect(res.body).toBeDefined()
-      expect(pg.queryMock).toHaveBeenCalledWith(`select "users"."name" as "users.name", "users"."age" as "users.age" from "users" where "users"."name" ilike $1 limit $2`, ["John%", 5000])
+      const expSql = `select "users"."name" as "users.name", "users"."age" as "users.age" from (select * from "users" where "users"."name" ilike $1 limit $2) as "users"`
+      expect(pg.queryMock).toHaveBeenCalledWith(expSql, ["John%", 5000])
     })
   })
 

packages/server/src/definitions/datasource.ts

@@ -1,3 +1,5 @@
+import {Table} from "./common";
+
 export enum Operation {
   CREATE = "CREATE",
   READ = "READ",
@@ -136,6 +138,9 @@ export interface QueryJson {
   sort?: SortJson
   paginate?: PaginationJson
   body?: object
+  meta?: {
+    table?: Table,
+  }
   extra?: {
     idFilter?: SearchFilters
   }

packages/server/src/integrations/base/sql.ts

@@ -1,7 +1,5 @@
 import { Knex, knex } from "knex"
 const BASE_LIMIT = 5000
-// if requesting a single row then need to up the limit for the sake of joins
-const SINGLE_ROW_LIMIT = 100
 import {
   QueryJson,
   SearchFilters,
@@ -146,46 +144,48 @@ function buildCreate(
 function buildRead(knex: Knex, json: QueryJson, limit: number): KnexQuery {
   let { endpoint, resource, filters, sort, paginate, relationships } = json
   const tableName = endpoint.entityId
-  let query: KnexQuery = knex(tableName)
   // select all if not specified
   if (!resource) {
     resource = { fields: [] }
   }
+  let selectStatement: string|string[] = "*"
   // handle select
   if (resource.fields && resource.fields.length > 0) {
     // select the resources as the format "table.columnName" - this is what is provided
     // by the resource builder further up
-    query = query.select(resource.fields.map(field => `${field} as ${field}`))
+    selectStatement = resource.fields.map(field => `${field} as ${field}`)
-  } else {
+  }
-    query = query.select("*")
+  let foundLimit = limit || BASE_LIMIT
+  // handle pagination
+  let foundOffset: number | null = null
+  if (paginate && paginate.page && paginate.limit) {
+    // @ts-ignore
+    const page = paginate.page <= 1 ? 0 : paginate.page - 1
+    const offset = page * paginate.limit
+    foundLimit = paginate.limit
+    foundOffset = offset
+  } else if (paginate && paginate.limit) {
+    foundLimit = paginate.limit
+  }
+  // start building the query
+  let query: KnexQuery = knex(tableName).limit(foundLimit)
+  if (foundOffset) {
+    query = query.offset(foundOffset)
   }
-  // handle where
-  query = addFilters(tableName, query, filters)
-  // handle join
-  query = addRelationships(query, tableName, relationships)
-  // handle sorting
   if (sort) {
     for (let [key, value] of Object.entries(sort)) {
       const direction = value === SortDirection.ASCENDING ? "asc" : "desc"
       query = query.orderBy(key, direction)
     }
   }
-  let foundLimit = limit || BASE_LIMIT
+  query = addFilters(tableName, query, filters)
-  // handle pagination
+  // @ts-ignore
-  if (paginate && paginate.page && paginate.limit) {
+  let preQuery: KnexQuery = knex({
     // @ts-ignore
-    const page = paginate.page <= 1 ? 0 : paginate.page - 1
+    [tableName]: query,
-    const offset = page * paginate.limit
+  }).select(selectStatement)
-    foundLimit = paginate.limit
+  // handle joins
-    query = query.offset(offset)
+  return addRelationships(preQuery, tableName, relationships)
-  } else if (paginate && paginate.limit) {
-    foundLimit = paginate.limit
-  }
-  if (foundLimit === 1) {
-    foundLimit = SINGLE_ROW_LIMIT
-  }
-  query = query.limit(foundLimit)
-  return query
 }
 
 function buildUpdate(

packages/server/src/integrations/mysql.ts

@@ -104,7 +104,7 @@ module MySQLModule {
     client: any,
     query: SqlQuery,
     connect: boolean = true
-  ): Promise<any[]> {
+  ): Promise<any[]|any> {
     // Node MySQL is callback based, so we must wrap our call in a promise
     return new Promise((resolve, reject) => {
       if (connect) {
@@ -238,6 +238,23 @@ module MySQLModule {
       return internalQuery(this.client, input, false)
     }
 
+    // when creating if an ID has been inserted need to make sure
+    // the id filter is enriched with it before trying to retrieve the row
+    checkLookupKeys(results: any, json: QueryJson) {
+      if (!results?.insertId || !json.meta?.table || !json.meta.table.primary) {
+        return json
+      }
+      const primaryKey = json.meta.table.primary?.[0]
+      json.extra = {
+        idFilter: {
+          equal: {
+            [primaryKey]: results.insertId
+          },
+        }
+      }
+      return json
+    }
+
     async query(json: QueryJson) {
       const operation = this._operation(json)
       this.client.connect()
@@ -250,7 +267,7 @@ module MySQLModule {
       const results = await internalQuery(this.client, input, false)
       // same as delete, manage returning
       if (operation === Operation.CREATE || operation === Operation.UPDATE) {
-        row = this.getReturningRow(json)
+        row = this.getReturningRow(this.checkLookupKeys(results, json))
       }
       this.client.end()
       if (operation !== Operation.READ) {

packages/server/src/integrations/postgres.ts

@@ -84,10 +84,13 @@ module PostgresModule {
     bigint: FieldTypes.NUMBER,
     decimal: FieldTypes.NUMBER,
     smallint: FieldTypes.NUMBER,
+    real: FieldTypes.NUMBER,
+    "double precision": FieldTypes.NUMBER,
     timestamp: FieldTypes.DATETIME,
     time: FieldTypes.DATETIME,
     boolean: FieldTypes.BOOLEAN,
     json: FieldTypes.JSON,
+    date: FieldTypes.DATETIME,
   }
 
   async function internalQuery(client: any, query: SqlQuery) {

packages/server/src/integrations/tests/sql.spec.js

@@ -57,7 +57,7 @@ describe("SQL query builder", () => {
     const query = sql._query(generateReadJson())
     expect(query).toEqual({
       bindings: [limit],
-      sql: `select * from "${TABLE_NAME}" limit $1`
+      sql: `select * from (select * from "${TABLE_NAME}" limit $1) as "${TABLE_NAME}"`
     })
   })
 
@@ -68,7 +68,7 @@ describe("SQL query builder", () => {
     }))
     expect(query).toEqual({
       bindings: [limit],
-      sql: `select "${TABLE_NAME}"."name" as "${nameProp}", "${TABLE_NAME}"."age" as "${ageProp}" from "${TABLE_NAME}" limit $1`
+      sql: `select "${TABLE_NAME}"."name" as "${nameProp}", "${TABLE_NAME}"."age" as "${ageProp}" from (select * from "${TABLE_NAME}" limit $1) as "${TABLE_NAME}"`
     })
   })
 
@@ -82,7 +82,7 @@ describe("SQL query builder", () => {
     }))
     expect(query).toEqual({
       bindings: ["John%", limit],
-      sql: `select * from "${TABLE_NAME}" where "${TABLE_NAME}"."name" ilike $1 limit $2`
+      sql: `select * from (select * from "${TABLE_NAME}" where "${TABLE_NAME}"."name" ilike $1 limit $2) as "${TABLE_NAME}"`
     })
   })
 
@@ -99,7 +99,7 @@ describe("SQL query builder", () => {
     }))
     expect(query).toEqual({
       bindings: [2, 10, limit],
-      sql: `select * from "${TABLE_NAME}" where "${TABLE_NAME}"."age" between $1 and $2 limit $3`
+      sql: `select * from (select * from "${TABLE_NAME}" where "${TABLE_NAME}"."age" between $1 and $2 limit $3) as "${TABLE_NAME}"`
     })
   })
 
@@ -115,7 +115,7 @@ describe("SQL query builder", () => {
     }))
     expect(query).toEqual({
       bindings: [10, "John", limit],
-      sql: `select * from "${TABLE_NAME}" where ("${TABLE_NAME}"."age" = $1) or ("${TABLE_NAME}"."name" = $2) limit $3`
+      sql: `select * from (select * from "${TABLE_NAME}" where ("${TABLE_NAME}"."age" = $1) or ("${TABLE_NAME}"."name" = $2) limit $3) as "${TABLE_NAME}"`
     })
   })
 
@@ -160,7 +160,7 @@ describe("SQL query builder", () => {
     const query = new Sql("mssql", 10)._query(generateReadJson())
     expect(query).toEqual({
       bindings: [10],
-      sql: `select top (@p0) * from [${TABLE_NAME}]`
+      sql: `select * from (select top (@p0) * from [${TABLE_NAME}]) as [${TABLE_NAME}]`
     })
   })
 
@@ -168,7 +168,7 @@ describe("SQL query builder", () => {
     const query = new Sql("mysql", 10)._query(generateReadJson())
     expect(query).toEqual({
       bindings: [10],
-      sql: `select * from \`${TABLE_NAME}\` limit ?`
+      sql: `select * from (select * from \`${TABLE_NAME}\` limit ?) as \`${TABLE_NAME}\``
     })
   })
 })

packages/string-templates/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@budibase/string-templates",
-  "version": "0.9.140-alpha.5",
+  "version": "0.9.140-alpha.10",
   "description": "Handlebars wrapper for Budibase templating.",
   "main": "src/index.cjs",
   "module": "dist/bundle.mjs",

packages/worker/package.json

@@ -1,7 +1,7 @@
 {
   "name": "@budibase/worker",
   "email": "hi@budibase.com",
-  "version": "0.9.140-alpha.5",
+  "version": "0.9.140-alpha.10",
   "description": "Budibase background service",
   "main": "src/index.js",
   "repository": {
@@ -25,8 +25,8 @@
   "author": "Budibase",
   "license": "AGPL-3.0-or-later",
   "dependencies": {
-    "@budibase/auth": "^0.9.140-alpha.5",
+    "@budibase/auth": "^0.9.140-alpha.10",
-    "@budibase/string-templates": "^0.9.140-alpha.5",
+    "@budibase/string-templates": "^0.9.140-alpha.10",
     "@koa/router": "^8.0.0",
     "@techpass/passport-openidconnect": "^0.3.0",
     "aws-sdk": "^2.811.0",