Merge branch 'master' into remove-automation-survey

packages/builder/src/components/start/CreateAppModal.svelte

@@ -26,6 +26,7 @@
   const values = writable({ name: "", url: null })
   const validation = createValidationStore()
   const encryptionValidation = createValidationStore()
+  const isEncryptedRegex = /^.*\.enc.*\.tar\.gz$/gm
 
   $: {
     const { url } = $values
@@ -37,7 +38,9 @@
     encryptionValidation.check({ ...$values })
   }
 
-  $: encryptedFile = $values.file?.name?.endsWith(".enc.tar.gz")
+  // filename should be separated to avoid updates everytime any other form element changes
+  $: filename = $values.file?.name
+  $: encryptedFile = isEncryptedRegex.test(filename)
 
   onMount(async () => {
     const lastChar = $auth.user?.firstName
@@ -171,7 +174,7 @@
           try {
             await createNewApp()
           } catch (error) {
-            notifications.error("Error creating app")
+            notifications.error(`Error creating app - ${error.message}`)
           }
         }
       },

packages/builder/src/pages/builder/portal/apps/index.svelte

@@ -139,7 +139,7 @@
       await auth.setInitInfo({})
       $goto(`/builder/app/${createdApp.instance._id}`)
     } catch (error) {
-      notifications.error("Error creating app")
+      notifications.error(`Error creating app - ${error.message}`)
     }
   }
 

packages/builder/src/stores/builder/datasources.ts

@@ -7,11 +7,26 @@ import {
 import { tables } from "./tables"
 import { queries } from "./queries"
 import { API } from "api"
-import { DatasourceFeature } from "@budibase/types"
+import {
+  DatasourceFeature,
+  Datasource,
+  Table,
+  Integration,
+  UIIntegration,
+  SourceName,
+} from "@budibase/types"
+// @ts-ignore
 import { TableNames } from "constants"
 
+// when building the internal DS - seems to represent it slightly differently to the backend typing of a DS
+interface InternalDatasource extends Omit<Datasource, "entities"> {
+  entities: Table[]
+}
+
 class TableImportError extends Error {
-  constructor(errors) {
+  errors: Record<string, string>
+
+  constructor(errors: Record<string, string>) {
     super()
     this.name = "TableImportError"
     this.errors = errors
@@ -26,8 +41,13 @@ class TableImportError extends Error {
   }
 }
 
+interface DatasourceStore {
+  list: Datasource[]
+  selectedDatasourceId: null | string
+}
+
 export function createDatasourcesStore() {
-  const store = writable({
+  const store = writable<DatasourceStore>({
     list: [],
     selectedDatasourceId: null,
   })
@@ -36,23 +56,25 @@ export function createDatasourcesStore() {
     // Set the internal datasource entities from the table list, which we're
     // able to keep updated unlike the egress generated definition of the
     // internal datasource
-    let internalDS = $store.list?.find(ds => ds._id === BUDIBASE_INTERNAL_DB_ID)
+    let internalDS: Datasource | InternalDatasource | undefined =
+      $store.list?.find(ds => ds._id === BUDIBASE_INTERNAL_DB_ID)
     let otherDS = $store.list?.filter(ds => ds._id !== BUDIBASE_INTERNAL_DB_ID)
     if (internalDS) {
+      const tables: Table[] = $tables.list?.filter((table: Table) => {
+        return (
+          table.sourceId === BUDIBASE_INTERNAL_DB_ID &&
+          table._id !== TableNames.USERS
+        )
+      })
       internalDS = {
         ...internalDS,
-        entities: $tables.list?.filter(table => {
+        entities: tables,
-          return (
-            table.sourceId === BUDIBASE_INTERNAL_DB_ID &&
-            table._id !== TableNames.USERS
-          )
-        }),
       }
     }
 
     // Build up enriched DS list
     // Only add the internal DS if we have at least one non-users table
-    let list = []
+    let list: (InternalDatasource | Datasource)[] = []
     if (internalDS?.entities?.length) {
       list.push(internalDS)
     }
@@ -75,62 +97,82 @@ export function createDatasourcesStore() {
     }))
   }
 
-  const select = id => {
+  const select = (id: string) => {
     store.update(state => ({
       ...state,
       selectedDatasourceId: id,
     }))
   }
 
-  const updateDatasource = (response, { ignoreErrors } = {}) => {
+  const updateDatasource = (
+    response: { datasource: Datasource; errors?: Record<string, string> },
+    { ignoreErrors }: { ignoreErrors?: boolean } = {}
+  ) => {
     const { datasource, errors } = response
     if (!ignoreErrors && errors && Object.keys(errors).length > 0) {
       throw new TableImportError(errors)
     }
-    replaceDatasource(datasource._id, datasource)
+    replaceDatasource(datasource._id!, datasource)
-    select(datasource._id)
+    select(datasource._id!)
     return datasource
   }
 
-  const updateSchema = async (datasource, tablesFilter) => {
+  const updateSchema = async (
-    const response = await API.buildDatasourceSchema({
+    datasource: Datasource,
-      datasourceId: datasource?._id,
+    tablesFilter: string[]
-      tablesFilter,
+  ) => {
-    })
+    const response = await API.buildDatasourceSchema(
+      datasource?._id!,
+      tablesFilter
+    )
     updateDatasource(response)
   }
 
-  const sourceCount = source => {
+  const sourceCount = (source: string) => {
     return get(store).list.filter(datasource => datasource.source === source)
       .length
   }
 
-  const checkDatasourceValidity = async (integration, datasource) => {
+  const checkDatasourceValidity = async (
+    integration: Integration,
+    datasource: Datasource
+  ): Promise<{ valid: boolean; error?: string }> => {
     if (integration.features?.[DatasourceFeature.CONNECTION_CHECKING]) {
       const { connected, error } = await API.validateDatasource(datasource)
       if (connected) {
-        return
+        return { valid: true }
+      } else {
+        return { valid: false, error }
       }
-
-      throw new Error(`Unable to connect: ${error}`)
     }
+    return { valid: true }
   }
 
-  const create = async ({ integration, config }) => {
+  const create = async ({
+    integration,
+    config,
+  }: {
+    integration: UIIntegration
+    config: Record<string, any>
+  }) => {
     const count = sourceCount(integration.name)
     const nameModifier = count === 0 ? "" : ` ${count + 1}`
 
-    const datasource = {
+    const datasource: Datasource = {
       type: "datasource",
-      source: integration.name,
+      source: integration.name as SourceName,
       config,
       name: `${integration.friendlyName}${nameModifier}`,
       plus: integration.plus && integration.name !== IntegrationTypes.REST,
       isSQL: integration.isSQL,
     }
 
-    if (await checkDatasourceValidity(integration, datasource)) {
+    const { valid, error } = await checkDatasourceValidity(
-      throw new Error("Unable to connect")
+      integration,
+      datasource
+    )
+    if (!valid) {
+      throw new Error(`Unable to connect - ${error}`)
     }
 
     const response = await API.createDatasource({
@@ -141,7 +183,13 @@ export function createDatasourcesStore() {
     return updateDatasource(response, { ignoreErrors: true })
   }
 
-  const update = async ({ integration, datasource }) => {
+  const update = async ({
+    integration,
+    datasource,
+  }: {
+    integration: Integration
+    datasource: Datasource
+  }) => {
     if (await checkDatasourceValidity(integration, datasource)) {
       throw new Error("Unable to connect")
     }
@@ -151,18 +199,15 @@ export function createDatasourcesStore() {
     return updateDatasource(response)
   }
 
-  const deleteDatasource = async datasource => {
+  const deleteDatasource = async (datasource: Datasource) => {
     if (!datasource?._id || !datasource?._rev) {
       return
     }
-    await API.deleteDatasource({
+    await API.deleteDatasource(datasource._id, datasource._rev)
-      datasourceId: datasource._id,
+    replaceDatasource(datasource._id)
-      datasourceRev: datasource._rev,
-    })
-    replaceDatasource(datasource._id, null)
   }
 
-  const replaceDatasource = (datasourceId, datasource) => {
+  const replaceDatasource = (datasourceId: string, datasource?: Datasource) => {
     if (!datasourceId) {
       return
     }
@@ -200,7 +245,7 @@ export function createDatasourcesStore() {
     }
   }
 
-  const getTableNames = async datasource => {
+  const getTableNames = async (datasource: Datasource) => {
     const info = await API.fetchInfoForDatasource(datasource)
     return info.tableNames || []
   }

packages/builder/src/stores/builder/sortedIntegrations.js

@@ -1,39 +0,0 @@
-import { integrations } from "./integrations"
-import { derived } from "svelte/store"
-
-import { DatasourceTypes } from "constants/backend"
-
-const getIntegrationOrder = type => {
-  if (type === DatasourceTypes.API) return 1
-  if (type === DatasourceTypes.RELATIONAL) return 2
-  if (type === DatasourceTypes.NON_RELATIONAL) return 3
-
-  // Sort all others arbitrarily by the first character of their name.
-  // Character codes can technically be as low as 0, so make sure the number is at least 4
-  return type.charCodeAt(0) + 4
-}
-
-export const createSortedIntegrationsStore = () => {
-  return derived(integrations, $integrations => {
-    const integrationsAsArray = Object.entries($integrations).map(
-      ([name, integration]) => ({
-        name,
-        ...integration,
-      })
-    )
-
-    return integrationsAsArray.sort((integrationA, integrationB) => {
-      const integrationASortOrder = getIntegrationOrder(integrationA.type)
-      const integrationBSortOrder = getIntegrationOrder(integrationB.type)
-      if (integrationASortOrder === integrationBSortOrder) {
-        return integrationA.friendlyName.localeCompare(
-          integrationB.friendlyName
-        )
-      }
-
-      return integrationASortOrder < integrationBSortOrder ? -1 : 1
-    })
-  })
-}
-
-export const sortedIntegrations = createSortedIntegrationsStore()

packages/builder/src/stores/builder/sortedIntegrations.ts

@@ -0,0 +1,46 @@
+import { integrations } from "./integrations"
+import { derived } from "svelte/store"
+
+import { DatasourceTypes } from "constants/backend"
+import { UIIntegration, Integration } from "@budibase/types"
+
+const getIntegrationOrder = (type: string | undefined) => {
+  // if type is not known, sort to end
+  if (!type) {
+    return Number.MAX_SAFE_INTEGER
+  }
+  if (type === DatasourceTypes.API) return 1
+  if (type === DatasourceTypes.RELATIONAL) return 2
+  if (type === DatasourceTypes.NON_RELATIONAL) return 3
+
+  // Sort all others arbitrarily by the first character of their name.
+  // Character codes can technically be as low as 0, so make sure the number is at least 4
+  return type.charCodeAt(0) + 4
+}
+
+export const createSortedIntegrationsStore = () => {
+  return derived<typeof integrations, UIIntegration[]>(
+    integrations,
+    $integrations => {
+      const entries: [string, Integration][] = Object.entries($integrations)
+      const integrationsAsArray = entries.map(([name, integration]) => ({
+        name,
+        ...integration,
+      }))
+
+      return integrationsAsArray.sort((integrationA, integrationB) => {
+        const integrationASortOrder = getIntegrationOrder(integrationA.type)
+        const integrationBSortOrder = getIntegrationOrder(integrationB.type)
+        if (integrationASortOrder === integrationBSortOrder) {
+          return integrationA.friendlyName.localeCompare(
+            integrationB.friendlyName
+          )
+        }
+
+        return integrationASortOrder < integrationBSortOrder ? -1 : 1
+      })
+    }
+  )
+}
+
+export const sortedIntegrations = createSortedIntegrationsStore()

packages/server/scripts/integrations/mysql/init.sql

@@ -30,6 +30,10 @@ CREATE TABLE Products (
     name text,
     updated time
 );
+CREATE TABLE `table with space` (
+    id serial primary key,
+    name text
+);
 INSERT INTO Persons (FirstName, LastName, Age, Address, City, CreatedAt) VALUES ('Mike', 'Hughes', 28.2, '123 Fake Street', 'Belfast', '2021-01-19 03:14:07');
 INSERT INTO Persons (FirstName, LastName, Age, Address, City, CreatedAt) VALUES ('Dave', 'Johnson', 29, '124 Fake Street', 'Belfast', '2022-04-01 00:11:11');
 INSERT INTO Person (Name) VALUES ('Elf');

packages/server/src/sdk/app/backups/imports.ts

@@ -187,6 +187,20 @@ export async function importApp(
       await decryptFiles(tmpPath, template.file.password)
     }
     const contents = await fsp.readdir(tmpPath)
+    const stillEncrypted = !!contents.find(name => name.endsWith(".enc"))
+    if (stillEncrypted) {
+      throw new Error("Files are encrypted but no password has been supplied.")
+    }
+    const isPlugin = !!contents.find(name => name === "plugin.min.js")
+    if (isPlugin) {
+      throw new Error("Supplied file is a plugin - cannot import as app.")
+    }
+    const isInvalid = !contents.find(name => name === DB_EXPORT_FILE)
+    if (isInvalid) {
+      throw new Error(
+        "App export does not appear to be valid - no DB file found."
+      )
+    }
     // have to handle object import
     if (contents.length && opts.importObjStoreContents) {
       let promises = []

packages/types/src/api/web/app/datasource.ts

@@ -12,7 +12,7 @@ export interface UpdateDatasourceResponse {
 export interface CreateDatasourceRequest {
   datasource: Datasource
   fetchSchema?: boolean
-  tablesFilter: string[]
+  tablesFilter?: string[]
 }
 
 export interface VerifyDatasourceRequest {

packages/types/src/index.ts

@@ -3,3 +3,4 @@ export * from "./sdk"
 export * from "./api"
 export * from "./core"
 export * from "./shared"
+export * from "./ui"

packages/types/src/ui/index.ts

@@ -0,0 +1 @@
+export * from "./stores"

packages/types/src/ui/stores/index.ts

@@ -0,0 +1 @@
+export * from "./integration"

packages/types/src/ui/stores/integration.ts

@@ -0,0 +1,5 @@
+import { Integration } from "@budibase/types"
+
+export interface UIIntegration extends Integration {
+  name: string
+}