Datasource plus - build schema

packages/builder/src/constants/index.js

@@ -29,6 +29,7 @@ export const IntegrationNames = {
   ARANGODB: "ArangoDB",
   ORACLE: "Oracle",
   GOOGLE_SHEETS: "Google Sheets",
+  SNOWFLAKE: "Snowflake",
 }
 
 // fields on the user table that cannot be edited

packages/server/src/integrations/snowflake.ts

@@ -1,12 +1,27 @@
-import { Integration, QueryTypes, SqlQuery } from "../definitions/datasource"
+import {
+  Integration,
+  QueryTypes,
+  SqlQuery,
+  DatasourceFieldTypes,
+} from "../definitions/datasource"
 import {
   SnowflakeError,
   Statement,
   createConnection,
   Connection,
 } from "snowflake-sdk"
+import {
+  SqlClients,
+  finaliseExternalTables,
+  buildExternalTableId,
+  convertSqlType,
+} from "./utils"
+import { DatasourcePlus } from "./base/datasourcePlus"
+import { Table, TableSchema } from "../definitions/common"
 
 module SnowflakeModule {
+  const Sql = require("./base/sql")
+
   interface SnowflakeConfig {
     account: string
     username: string
@@ -17,33 +32,34 @@ module SnowflakeModule {
   }
 
   const SCHEMA: Integration = {
+    plus: true,
     docs: "https://developers.snowflake.com/",
     description:
       "Snowflake is a solution for data warehousing, data lakes, data engineering, data science, data application development, and securely sharing and consuming shared data.",
     friendlyName: "Snowflake",
     datasource: {
       account: {
-        type: "string",
+        type: DatasourceFieldTypes.STRING,
         required: true,
       },
       username: {
-        type: "string",
+        type: DatasourceFieldTypes.STRING,
         required: true,
       },
       password: {
-        type: "password",
+        type: DatasourceFieldTypes.PASSWORD,
         required: true,
       },
       warehouse: {
-        type: "string",
+        type: DatasourceFieldTypes.STRING,
         required: true,
       },
       database: {
-        type: "string",
+        type: DatasourceFieldTypes.STRING,
         required: true,
       },
       schema: {
-        type: "string",
+        type: DatasourceFieldTypes.STRING,
         required: true,
       },
     },
@@ -63,13 +79,77 @@ module SnowflakeModule {
     },
   }
 
-  class SnowflakeIntegration {
+  class SnowflakeIntegration extends Sql implements DatasourcePlus {
     private client: Connection
+    private config: SnowflakeConfig
+    public tables: Record<string, Table> = {}
+    public schemaErrors: Record<string, string> = {}
 
     constructor(config: SnowflakeConfig) {
+      super(SqlClients.SNOWFLAKE)
+      this.config = config
       this.client = createConnection(config)
     }
 
+    getBindingIdentifier(): string {
+      return "?"
+    }
+
+    getStringConcat(parts: string[]): string {
+      return `concat(${parts.join(", ")})`
+    }
+
+    async buildSchema(datasourceId: string, entities: Record<string, Table>) {
+      const tables: { [key: string]: Table } = {}
+      const database = this.config.database
+
+      // get the tables first
+      const tablesResp = await this.internalQuery({ sql: "SHOW TABLES;" })
+      const tableNames = tablesResp.map((obj: any) => obj.name)
+      for (let tableName of tableNames) {
+        const primaryKeys = []
+        const schema: TableSchema = {}
+        const descResp = await this.internalQuery({
+          sql: `DESCRIBE TABLE ${tableName};`,
+        })
+        if (tableName === "CUSTOMER") {
+          console.log("DESC = ", descResp)
+        }
+        for (let column of descResp) {
+          const columnName = column.Field
+          if (
+            column["primary key"] === "Y" &&
+            primaryKeys.indexOf(column.Key) === -1
+          ) {
+            primaryKeys.push(columnName)
+          }
+          const constraints = {
+            presence: column["null?"] !== "Y",
+          }
+          const isAuto: boolean = column.default
+            ?.toLowerCase()
+            .includes("increment")
+          schema[columnName] = {
+            name: columnName,
+            autocolumn: isAuto,
+            constraints,
+            ...convertSqlType(column["type"]),
+          }
+        }
+        if (!tables[tableName]) {
+          tables[tableName] = {
+            _id: buildExternalTableId(datasourceId, tableName),
+            primary: primaryKeys,
+            name: tableName,
+            schema,
+          }
+        }
+      }
+      const final = finaliseExternalTables(tables, entities)
+      this.tables = final.tables
+      this.schemaErrors = final.errors
+    }
+
     async connectAsync() {
       return new Promise((resolve, reject) => {
         this.client.connect(function (err: any, conn: any) {
@@ -80,7 +160,9 @@ module SnowflakeModule {
     }
 
     async internalQuery(query: SqlQuery) {
-      await this.connectAsync()
+      if (!this.client.isUp()) {
+        await this.connectAsync()
+      }
       let response: any = await new Promise((resolve, reject) =>
         this.client.execute({
           sqlText: query.sql,

packages/server/src/integrations/utils.ts

@@ -73,6 +73,7 @@ export enum SqlClients {
   POSTGRES = "pg",
   MY_SQL = "mysql2",
   ORACLE = "oracledb",
+  SNOWFLAKE = "snowflake-sdk",
 }
 
 export function isExternalTable(tableId: string) {
@@ -173,6 +174,7 @@ export function isSQL(datasource: Datasource): boolean {
     SourceNames.SQL_SERVER,
     SourceNames.MYSQL,
     SourceNames.ORACLE,
+    SourceNames.SNOWFLAKE,
   ]
   return SQL.indexOf(datasource.source) !== -1
 }