Merge branch 'master' of github.com:Budibase/budibase into develop

lerna.json

@@ -1,5 +1,5 @@
 {
-  "version": "2.8.18-alpha.5",
+  "version": "2.8.21",
   "npmClient": "yarn",
   "packages": [
     "packages/*"
@@ -19,4 +19,4 @@
       "loadEnvFiles": false
     }
   }
-}
+}

packages/bbui/src/Icon/Icon.svelte

@@ -47,7 +47,7 @@
   </svg>
   {#if tooltip && showTooltip}
     <div class="tooltip" in:fade={{ duration: 130, delay: 250 }}>
-      <Tooltip textWrapping direction="bottom" text={tooltip} />
+      <Tooltip textWrapping direction="top" text={tooltip} />
     </div>
   {/if}
 </div>
@@ -80,15 +80,14 @@
     position: absolute;
     pointer-events: none;
     left: 50%;
-    top: calc(100% + 4px);
+    bottom: calc(100% + 4px);
-    width: 100vw;
-    max-width: 150px;
     transform: translateX(-50%);
     text-align: center;
+    z-index: 1;
   }
 
   .spectrum-Icon--sizeXS {
-    width: 10px;
+    width: var(--spectrum-global-dimension-size-150);
-    height: 10px;
+    height: var(--spectrum-global-dimension-size-150);
   }
 </style>

packages/builder/src/components/backend/DataTable/modals/CreateEditColumn.svelte

@@ -33,6 +33,7 @@
   import { getBindings } from "components/backend/DataTable/formula"
   import { getContext } from "svelte"
   import JSONSchemaModal from "./JSONSchemaModal.svelte"
+  import { ValidColumnNameRegex } from "@budibase/shared-core"
 
   const AUTO_TYPE = "auto"
   const FORMULA_TYPE = FIELDS.FORMULA.type
@@ -375,7 +376,7 @@
     const newError = {}
     if (!external && fieldInfo.name?.startsWith("_")) {
       newError.name = `Column name cannot start with an underscore.`
-    } else if (fieldInfo.name && !fieldInfo.name.match(/^[_a-zA-Z0-9\s]*$/g)) {
+    } else if (fieldInfo.name && !fieldInfo.name.match(ValidColumnNameRegex)) {
       newError.name = `Illegal character; must be alpha-numeric.`
     } else if (PROHIBITED_COLUMN_NAMES.some(name => fieldInfo.name === name)) {
       newError.name = `${PROHIBITED_COLUMN_NAMES.join(

packages/builder/src/components/backend/TableNavigator/TableDataImport.svelte

@@ -1,17 +1,9 @@
 <script>
-  import { Select } from "@budibase/bbui"
+  import { Select, Icon } from "@budibase/bbui"
   import { FIELDS } from "constants/backend"
   import { API } from "api"
   import { parseFile } from "./utils"
 
-  let fileInput
-  let error = null
-  let fileName = null
-
-  let loading = false
-  let validation = {}
-  let validateHash = ""
-
   export let rows = []
   export let schema = {}
   export let allValid = true
@@ -49,6 +41,28 @@
     },
   ]
 
+  let fileInput
+  let error = null
+  let fileName = null
+  let fileType = null
+  let loading = false
+  let validation = {}
+  let validateHash = ""
+  let errors = {}
+
+  $: displayColumnOptions = Object.keys(schema || {}).filter(column => {
+    return validation[column]
+  })
+  $: {
+    // binding in consumer is causing double renders here
+    const newValidateHash = JSON.stringify(rows) + JSON.stringify(schema)
+    if (newValidateHash !== validateHash) {
+      validate(rows, schema)
+    }
+    validateHash = newValidateHash
+  }
+  $: openFileUpload(promptUpload, fileInput)
+
   async function handleFile(e) {
     loading = true
     error = null
@@ -67,34 +81,23 @@
 
   async function validate(rows, schema) {
     loading = true
-    error = null
-    validation = {}
-    allValid = false
-
     try {
       if (rows.length > 0) {
         const response = await API.validateNewTableImport({ rows, schema })
         validation = response.schemaValidation
         allValid = response.allValid
+        errors = response.errors
+        error = null
       }
     } catch (e) {
       error = e.message
+      validation = {}
+      allValid = false
+      errors = {}
     }
-
     loading = false
   }
 
-  $: {
-    // binding in consumer is causing double renders here
-    const newValidateHash = JSON.stringify(rows) + JSON.stringify(schema)
-
-    if (newValidateHash !== validateHash) {
-      validate(rows, schema)
-    }
-
-    validateHash = newValidateHash
-  }
-
   const handleChange = (name, e) => {
     schema[name].type = e.detail
     schema[name].constraints = FIELDS[e.detail.toUpperCase()].constraints
@@ -106,7 +109,13 @@
     }
   }
 
-  $: openFileUpload(promptUpload, fileInput)
+  const deleteColumn = name => {
+    if (loading) {
+      return
+    }
+    delete schema[name]
+    schema = schema
+  }
 </script>
 
 <div class="dropzone">
@@ -119,10 +128,8 @@
     on:change={handleFile}
   />
   <label for="file-upload" class:uploaded={rows.length > 0}>
-    {#if loading}
+    {#if error}
-      loading...
+      Error: {error}
-    {:else if error}
-      error: {error}
     {:else if fileName}
       {fileName}
     {:else}
@@ -142,23 +149,26 @@
           placeholder={null}
           getOptionLabel={option => option.label}
           getOptionValue={option => option.value}
-          disabled={loading}
         />
         <span
-          class={loading || validation[column.name]
+          class={validation[column.name]
             ? "fieldStatusSuccess"
             : "fieldStatusFailure"}
         >
-          {validation[column.name] ? "Success" : "Failure"}
+          {#if validation[column.name]}
+            Success
+          {:else}
+            Failure
+            {#if errors[column.name]}
+              <Icon name="Help" tooltip={errors[column.name]} />
+            {/if}
+          {/if}
         </span>
-        <i
+        <Icon
-          class={`omit-button ri-close-circle-fill ${
+          size="S"
-            loading ? "omit-button-disabled" : ""
+          name="Close"
-          }`}
+          hoverable
-          on:click={() => {
+          on:click={() => deleteColumn(column.name)}
-            delete schema[column.name]
-            schema = schema
-          }}
         />
       </div>
     {/each}
@@ -167,7 +177,7 @@
     <Select
       label="Display Column"
       bind:value={displayColumn}
-      options={Object.keys(schema)}
+      options={displayColumnOptions}
       sort
     />
   </div>
@@ -235,23 +245,16 @@
     justify-self: center;
     font-weight: 600;
   }
-
   .fieldStatusFailure {
     color: var(--red);
     justify-self: center;
     font-weight: 600;
+    display: flex;
+    align-items: center;
+    gap: 4px;
   }
-
+  .fieldStatusFailure :global(.spectrum-Icon) {
-  .omit-button {
+    width: 12px;
-    font-size: 1.2em;
-    color: var(--grey-7);
-    cursor: pointer;
-    justify-self: flex-end;
-  }
-
-  .omit-button-disabled {
-    pointer-events: none;
-    opacity: 70%;
   }
 
   .display-column {

packages/builder/src/components/integration/RestQueryViewer.svelte

@@ -419,16 +419,22 @@
     if (query && !query.fields.pagination) {
       query.fields.pagination = {}
     }
-    dynamicVariables = getDynamicVariables(
+    // if query doesn't have ID then its new - don't try to copy existing dynamic variables
-      datasource,
+    if (!queryId) {
-      query._id,
+      dynamicVariables = []
-      (variable, queryId) => variable.queryId === queryId
+      globalDynamicBindings = getDynamicVariables(datasource)
-    )
+    } else {
-    globalDynamicBindings = getDynamicVariables(
+      dynamicVariables = getDynamicVariables(
-      datasource,
+        datasource,
-      query._id,
+        query._id,
-      (variable, queryId) => variable.queryId !== queryId
+        (variable, queryId) => variable.queryId === queryId
-    )
+      )
+      globalDynamicBindings = getDynamicVariables(
+        datasource,
+        query._id,
+        (variable, queryId) => variable.queryId !== queryId
+      )
+    }
 
     prettifyQueryRequestBody(
       query,

packages/builder/src/pages/builder/app/[application]/data/datasource/[datasourceId]/_components/panels/Variables/ViewDynamicVariables.svelte

@@ -18,7 +18,7 @@
   const onClick = dynamicVariable => {
     const queryId = dynamicVariable.queryId
     queries.select({ _id: queryId })
-    $goto(`./${queryId}`)
+    $goto(`../../query/${queryId}`)
   }
 
   /**

packages/server/src/api/controllers/datasource.ts

@@ -1,34 +1,33 @@
 import {
-  generateDatasourceID,
-  getDatasourceParams,
-  getQueryParams,
   DocumentType,
-  BudibaseInternalDB,
+  generateDatasourceID,
+  getQueryParams,
   getTableParams,
 } from "../../db/utils"
 import { destroy as tableDestroy } from "./table/internal"
 import { BuildSchemaErrors, InvalidColumns } from "../../constants"
 import { getIntegration } from "../../integrations"
 import { invalidateDynamicVariables } from "../../threads/utils"
-import { db as dbCore, context, events } from "@budibase/backend-core"
+import { context, db as dbCore, events } from "@budibase/backend-core"
 import {
-  UserCtx,
-  Datasource,
-  Row,
-  CreateDatasourceResponse,
-  UpdateDatasourceResponse,
   CreateDatasourceRequest,
-  VerifyDatasourceRequest,
+  CreateDatasourceResponse,
-  VerifyDatasourceResponse,
+  Datasource,
+  DatasourcePlus,
   FetchDatasourceInfoRequest,
   FetchDatasourceInfoResponse,
   IntegrationBase,
-  DatasourcePlus,
+  RestConfig,
   SourceName,
+  UpdateDatasourceResponse,
+  UserCtx,
+  VerifyDatasourceRequest,
+  VerifyDatasourceResponse,
 } from "@budibase/types"
 import sdk from "../../sdk"
 import { builderSocket } from "../../websockets"
 import { setupCreationAuth as googleSetupCreationAuth } from "../../integrations/googlesheets"
+import { areRESTVariablesValid } from "../../sdk/app/datasources/datasources"
 
 function getErrorTables(errors: any, errorType: string) {
   return Object.entries(errors)
@@ -119,46 +118,7 @@ async function buildFilteredSchema(datasource: Datasource, filter?: string[]) {
 }
 
 export async function fetch(ctx: UserCtx) {
-  // Get internal tables
+  ctx.body = await sdk.datasources.fetch()
-  const db = context.getAppDB()
-  const internalTables = await db.allDocs(
-    getTableParams(null, {
-      include_docs: true,
-    })
-  )
-
-  const internal = internalTables.rows.reduce((acc: any, row: Row) => {
-    const sourceId = row.doc.sourceId || "bb_internal"
-    acc[sourceId] = acc[sourceId] || []
-    acc[sourceId].push(row.doc)
-    return acc
-  }, {})
-
-  const bbInternalDb = {
-    ...BudibaseInternalDB,
-  }
-
-  // Get external datasources
-  const datasources = (
-    await db.allDocs(
-      getDatasourceParams(null, {
-        include_docs: true,
-      })
-    )
-  ).rows.map(row => row.doc)
-
-  const allDatasources: Datasource[] = await sdk.datasources.removeSecrets([
-    bbInternalDb,
-    ...datasources,
-  ])
-
-  for (let datasource of allDatasources) {
-    if (datasource.type === dbCore.BUDIBASE_DATASOURCE_TYPE) {
-      datasource.entities = internal[datasource._id!]
-    }
-  }
-
-  ctx.body = [bbInternalDb, ...datasources]
 }
 
 export async function verify(
@@ -290,6 +250,14 @@ export async function update(ctx: UserCtx<any, UpdateDatasourceResponse>) {
     datasource.config!.auth = auth
   }
 
+  // check all variables are unique
+  if (
+    datasource.source === SourceName.REST &&
+    !sdk.datasources.areRESTVariablesValid(datasource)
+  ) {
+    ctx.throw(400, "Duplicate dynamic/static variable names are invalid.")
+  }
+
   const response = await db.put(
     sdk.tables.populateExternalTableSchemas(datasource)
   )

packages/server/src/api/controllers/query/index.ts

@@ -1,4 +1,4 @@
-import { generateQueryID, getQueryParams, isProdAppID } from "../../../db/utils"
+import { generateQueryID } from "../../../db/utils"
 import { BaseQueryVerbs, FieldTypes } from "../../../constants"
 import { Thread, ThreadType } from "../../../threads"
 import { save as saveDatasource } from "../datasource"
@@ -28,15 +28,7 @@ function enrichQueries(input: any) {
 }
 
 export async function fetch(ctx: any) {
-  const db = context.getAppDB()
+  ctx.body = await sdk.queries.fetch()
-
-  const body = await db.allDocs(
-    getQueryParams(null, {
-      include_docs: true,
-    })
-  )
-
-  ctx.body = enrichQueries(body.rows.map((row: any) => row.doc))
 }
 
 const _import = async (ctx: any) => {
@@ -103,14 +95,8 @@ export async function save(ctx: any) {
 }
 
 export async function find(ctx: any) {
-  const db = context.getAppDB()
+  const queryId = ctx.params.queryId
-  const query = enrichQueries(await db.get(ctx.params.queryId))
+  ctx.body = await sdk.queries.find(queryId)
-  // remove properties that could be dangerous in real app
-  if (isProdAppID(ctx.appId)) {
-    delete query.fields
-    delete query.parameters
-  }
-  ctx.body = query
 }
 
 //Required to discern between OIDC OAuth config entries

packages/server/src/sdk/app/datasources/datasources.ts

@@ -1,4 +1,4 @@
-import { context } from "@budibase/backend-core"
+import { context, db as dbCore } from "@budibase/backend-core"
 import { findHBSBlocks, processObjectSync } from "@budibase/string-templates"
 import {
   Datasource,
@@ -8,15 +8,88 @@ import {
   RestAuthConfig,
   RestAuthType,
   RestBasicAuthConfig,
+  Row,
+  RestConfig,
   SourceName,
 } from "@budibase/types"
 import { cloneDeep } from "lodash/fp"
 import { getEnvironmentVariables } from "../../utils"
 import { getDefinitions, getDefinition } from "../../../integrations"
 import _ from "lodash"
+import {
+  BudibaseInternalDB,
+  getDatasourceParams,
+  getTableParams,
+} from "../../../db/utils"
+import sdk from "../../index"
 
 const ENV_VAR_PREFIX = "env."
 
+export async function fetch() {
+  // Get internal tables
+  const db = context.getAppDB()
+  const internalTables = await db.allDocs(
+    getTableParams(null, {
+      include_docs: true,
+    })
+  )
+
+  const internal = internalTables.rows.reduce((acc: any, row: Row) => {
+    const sourceId = row.doc.sourceId || "bb_internal"
+    acc[sourceId] = acc[sourceId] || []
+    acc[sourceId].push(row.doc)
+    return acc
+  }, {})
+
+  const bbInternalDb = {
+    ...BudibaseInternalDB,
+  }
+
+  // Get external datasources
+  const datasources = (
+    await db.allDocs(
+      getDatasourceParams(null, {
+        include_docs: true,
+      })
+    )
+  ).rows.map(row => row.doc)
+
+  const allDatasources: Datasource[] = await sdk.datasources.removeSecrets([
+    bbInternalDb,
+    ...datasources,
+  ])
+
+  for (let datasource of allDatasources) {
+    if (datasource.type === dbCore.BUDIBASE_DATASOURCE_TYPE) {
+      datasource.entities = internal[datasource._id!]
+    }
+  }
+
+  return [bbInternalDb, ...datasources]
+}
+
+export function areRESTVariablesValid(datasource: Datasource) {
+  const restConfig = datasource.config as RestConfig
+  const varNames: string[] = []
+  if (restConfig.dynamicVariables) {
+    for (let variable of restConfig.dynamicVariables) {
+      if (varNames.includes(variable.name)) {
+        return false
+      }
+      varNames.push(variable.name)
+    }
+  }
+  if (restConfig.staticVariables) {
+    for (let name of Object.keys(restConfig.staticVariables)) {
+      if (varNames.includes(name)) {
+        return false
+      }
+      varNames.push(name)
+    }
+  }
+  return true
+}
+
 export function checkDatasourceTypes(schema: Integration, config: any) {
   for (let key of Object.keys(config)) {
     if (!schema.datasource[key]) {

packages/server/src/sdk/app/queries/queries.ts

@@ -1,5 +1,49 @@
 import { getEnvironmentVariables } from "../../utils"
 import { processStringSync } from "@budibase/string-templates"
+import { context } from "@budibase/backend-core"
+import { getQueryParams, isProdAppID } from "../../../db/utils"
+import { BaseQueryVerbs } from "../../../constants"
+
+// simple function to append "readable" to all read queries
+function enrichQueries(input: any) {
+  const wasArray = Array.isArray(input)
+  const queries = wasArray ? input : [input]
+  for (let query of queries) {
+    if (query.queryVerb === BaseQueryVerbs.READ) {
+      query.readable = true
+    }
+  }
+  return wasArray ? queries : queries[0]
+}
+
+export async function find(queryId: string) {
+  const db = context.getAppDB()
+  const appId = context.getAppId()
+  const query = enrichQueries(await db.get(queryId))
+  // remove properties that could be dangerous in real app
+  if (isProdAppID(appId)) {
+    delete query.fields
+    delete query.parameters
+  }
+  return query
+}
+
+export async function fetch(opts: { enrich: boolean } = { enrich: true }) {
+  const db = context.getAppDB()
+
+  const body = await db.allDocs(
+    getQueryParams(null, {
+      include_docs: true,
+    })
+  )
+
+  const queries = body.rows.map((row: any) => row.doc)
+  if (opts.enrich) {
+    return enrichQueries(queries)
+  } else {
+    return queries
+  }
+}
 
 export async function enrichContext(
   fields: Record<string, any>,

packages/server/src/utilities/schema.ts

@@ -1,4 +1,5 @@
 import { FieldTypes } from "../constants"
+import { ValidColumnNameRegex } from "@budibase/shared-core"
 
 interface SchemaColumn {
   readonly name: string
@@ -27,6 +28,7 @@ interface ValidationResults {
   schemaValidation: SchemaValidation
   allValid: boolean
   invalidColumns: Array<string>
+  errors: Record<string, string>
 }
 
 const PARSERS: any = {
@@ -69,6 +71,7 @@ export function validate(rows: Rows, schema: Schema): ValidationResults {
     schemaValidation: {},
     allValid: false,
     invalidColumns: [],
+    errors: {},
   }
 
   rows.forEach(row => {
@@ -79,6 +82,11 @@ export function validate(rows: Rows, schema: Schema): ValidationResults {
       // If the columnType is not a string, then it's not present in the schema, and should be added to the invalid columns array
       if (typeof columnType !== "string") {
         results.invalidColumns.push(columnName)
+      } else if (!columnName.match(ValidColumnNameRegex)) {
+        // Check for special characters in column names
+        results.schemaValidation[columnName] = false
+        results.errors[columnName] =
+          "Column names can't contain special characters"
       } else if (
         columnData == null &&
         !schema[columnName].constraints?.presence

packages/shared-core/src/constants.ts

@@ -94,3 +94,4 @@ export enum BuilderSocketEvent {
 }
 
 export const SocketSessionTTL = 60
+export const ValidColumnNameRegex = /^[_a-zA-Z0-9\s]*$/g

packages/types/src/documents/app/datasource.ts

@@ -7,9 +7,7 @@ export interface Datasource extends Document {
   name?: string
   source: SourceName
   // the config is defined by the schema
-  config?: {
+  config?: Record<string, any>
-    [key: string]: string | number | boolean | any[]
-  }
   plus?: boolean
   entities?: {
     [key: string]: Table