Remove elasticsearch global mock, fix tests.

packages/server/__mocks__/@elastic/elasticsearch.ts

@@ -1,24 +0,0 @@
-const elastic: any = {}
-
-elastic.Client = function () {
-  this.index = jest.fn().mockResolvedValue({ body: [] })
-  this.search = jest.fn().mockResolvedValue({
-    body: {
-      hits: {
-        hits: [
-          {
-            _source: {
-              name: "test",
-            },
-          },
-        ],
-      },
-    },
-  })
-  this.update = jest.fn().mockResolvedValue({ body: [] })
-  this.delete = jest.fn().mockResolvedValue({ body: [] })
-
-  this.close = jest.fn()
-}
-
-module.exports = elastic

packages/server/datasource-sha.env

@@ -3,3 +3,4 @@ MYSQL_SHA=sha256:9de9d54fecee6253130e65154b930978b1fcc336bcc86dfd06e89b72a2588eb
 POSTGRES_SHA=sha256:bd0d8e485d1aca439d39e5ea99b931160bd28d862e74c786f7508e9d0053090e
 MONGODB_SHA=sha256:afa36bca12295b5f9dae68a493c706113922bdab520e901bd5d6c9d7247a1d8d
 MARIADB_SHA=sha256:e59ba8783bf7bc02a4779f103bb0d8751ac0e10f9471089709608377eded7aa8
+ELASTICSEARCH_SHA=sha256:9a6443f55243f6acbfeb4a112d15eb3b9aac74bf25e0e39fa19b3ddd3a6879d0

packages/server/src/integrations/elasticsearch.ts

@@ -10,7 +10,7 @@ import {
 import { Client, ClientOptions } from "@elastic/elasticsearch"
 import { HOST_ADDRESS } from "./utils"
 
-interface ElasticsearchConfig {
+export interface ElasticsearchConfig {
   url: string
   ssl?: boolean
   ca?: string
@@ -99,9 +99,9 @@ const SCHEMA: Integration = {
   },
 }
 
-class ElasticSearchIntegration implements IntegrationBase {
+export class ElasticSearchIntegration implements IntegrationBase {
   private config: ElasticsearchConfig
-  private client
+  private client: Client
 
   constructor(config: ElasticsearchConfig) {
     this.config = config
@@ -132,20 +132,23 @@ class ElasticSearchIntegration implements IntegrationBase {
     }
   }
 
-  async create(query: { index: string; json: object }) {
-    const { index, json } = query
+  async create(query: {
+    index: string
+    json: object
+    extra?: Record<string, string>
+  }) {
+    const { index, json, extra } = query
 
     try {
       const result = await this.client.index({
         index,
         body: json,
+        ...extra,
       })
       return result.body
     } catch (err) {
       console.error("Error writing to elasticsearch", err)
       throw err
-    } finally {
-      await this.client.close()
     }
   }
 
@@ -160,41 +163,46 @@ class ElasticSearchIntegration implements IntegrationBase {
     } catch (err) {
       console.error("Error querying elasticsearch", err)
       throw err
-    } finally {
-      await this.client.close()
     }
   }
 
-  async update(query: { id: string; index: string; json: object }) {
-    const { id, index, json } = query
+  async update(query: {
+    id: string
+    index: string
+    json: object
+    extra?: Record<string, string>
+  }) {
+    const { id, index, json, extra } = query
     try {
       const result = await this.client.update({
         id,
         index,
         body: json,
+        ...extra,
       })
       return result.body
     } catch (err) {
       console.error("Error querying elasticsearch", err)
       throw err
-    } finally {
-      await this.client.close()
     }
   }
 
-  async delete(query: { id: string; index: string }) {
-    const { id, index } = query
+  async delete(query: {
+    id: string
+    index: string
+    extra?: Record<string, string>
+  }) {
+    const { id, index, extra } = query
     try {
       const result = await this.client.delete({
         id,
         index,
+        ...extra,
       })
       return result.body
     } catch (err) {
       console.error("Error deleting from elasticsearch", err)
       throw err
-    } finally {
-      await this.client.close()
     }
   }
 }

packages/server/src/integrations/tests/elasticsearch.spec.ts

@@ -1,83 +1,76 @@
-import { default as ElasticSearchIntegration } from "../elasticsearch"
-
-jest.mock("@elastic/elasticsearch")
-
-class TestConfiguration {
-  integration: any
-
-  constructor(config: any = {}) {
-    this.integration = new ElasticSearchIntegration.integration(config)
-  }
-}
+import { Datasource } from "@budibase/types"
+import { ElasticsearchConfig, ElasticSearchIntegration } from "../elasticsearch"
+import * as elasticsearch from "../tests/utils/elasticsearch"
+import { generator } from "@budibase/backend-core/tests"
 
 describe("Elasticsearch Integration", () => {
-  let config: any
-  let indexName = "Users"
+  let datasource: Datasource
+  let integration: ElasticSearchIntegration
+
+  let index: string
+
+  beforeAll(async () => {
+    datasource = await elasticsearch.getDatasource()
+  })
 
   beforeEach(() => {
-    config = new TestConfiguration()
+    index = generator.guid()
+    integration = new ElasticSearchIntegration(
+      datasource.config! as ElasticsearchConfig
+    )
   })
 
-  it("calls the create method with the correct params", async () => {
-    const body = {
-      name: "Hello",
-    }
-    await config.integration.create({
-      index: indexName,
-      json: body,
+  it("can create a record", async () => {
+    await integration.create({
+      index,
+      json: { name: "Hello" },
+      extra: { refresh: "true" },
     })
-    expect(config.integration.client.index).toHaveBeenCalledWith({
-      index: indexName,
-      body,
+    const records = await integration.read({
+      index,
+      json: { query: { match_all: {} } },
     })
+    expect(records).toEqual([{ name: "Hello" }])
   })
 
-  it("calls the read method with the correct params", async () => {
-    const body = {
-      query: {
-        term: {
-          name: "kimchy",
-        },
-      },
-    }
-    const response = await config.integration.read({
-      index: indexName,
-      json: body,
+  it("can update a record", async () => {
+    const create = await integration.create({
+      index,
+      json: { name: "Hello" },
+      extra: { refresh: "true" },
     })
-    expect(config.integration.client.search).toHaveBeenCalledWith({
-      index: indexName,
-      body,
+
+    await integration.update({
+      id: create._id,
+      index,
+      json: { doc: { name: "World" } },
+      extra: { refresh: "true" },
     })
-    expect(response).toEqual(expect.any(Array))
+
+    const records = await integration.read({
+      index,
+      json: { query: { match_all: {} } },
+    })
+    expect(records).toEqual([{ name: "World" }])
   })
 
-  it("calls the update method with the correct params", async () => {
-    const body = {
-      name: "updated",
-    }
-
-    const response = await config.integration.update({
-      id: "1234",
-      index: indexName,
-      json: body,
+  it("can delete a record", async () => {
+    const create = await integration.create({
+      index,
+      json: { name: "Hello" },
+      extra: { refresh: "true" },
     })
 
-    expect(config.integration.client.update).toHaveBeenCalledWith({
-      id: "1234",
-      index: indexName,
-      body,
+    await integration.delete({
+      id: create._id,
+      index,
+      extra: { refresh: "true" },
     })
-    expect(response).toEqual(expect.any(Array))
-  })
 
-  it("calls the delete method with the correct params", async () => {
-    const body = {
-      id: "1234",
-    }
-
-    const response = await config.integration.delete(body)
-
-    expect(config.integration.client.delete).toHaveBeenCalledWith(body)
-    expect(response).toEqual(expect.any(Array))
+    const records = await integration.read({
+      index,
+      json: { query: { match_all: {} } },
+    })
+    expect(records).toEqual([])
   })
 })

packages/server/src/integrations/tests/utils/elasticsearch.ts

@@ -0,0 +1,43 @@
+import { Datasource, SourceName } from "@budibase/types"
+import { GenericContainer, Wait } from "testcontainers"
+import { testContainerUtils } from "@budibase/backend-core/tests"
+import { startContainer } from "."
+import { ELASTICSEARCH_IMAGE } from "./images"
+import { ElasticsearchConfig } from "../../elasticsearch"
+
+let ports: Promise<testContainerUtils.Port[]>
+
+export async function getDatasource(): Promise<Datasource> {
+  if (!ports) {
+    ports = startContainer(
+      new GenericContainer(ELASTICSEARCH_IMAGE)
+        .withExposedPorts(9200)
+        .withEnvironment({
+          "discovery.type": "single-node",
+          "xpack.security.enabled": "false",
+        })
+        .withWaitStrategy(
+          Wait.forHttp(
+            "/_cluster/health?wait_for_status=yellow&timeout=10s",
+            9200
+          ).withStartupTimeout(60000)
+        )
+        .withTmpFs({ "/usr/share/elasticsearch/data": "rw" })
+    )
+  }
+
+  const port = (await ports).find(x => x.container === 9200)?.host
+  if (!port) {
+    throw new Error("Elasticsearch port not found")
+  }
+
+  const config: ElasticsearchConfig = {
+    url: `http://127.0.0.1:${port}`,
+  }
+
+  return {
+    type: "datasource",
+    source: SourceName.ELASTICSEARCH,
+    config,
+  }
+}

packages/server/src/integrations/tests/utils/images.ts

@@ -12,3 +12,4 @@ export const POSTGRES_IMAGE = `postgres@${process.env.POSTGRES_SHA}`
 export const POSTGRES_LEGACY_IMAGE = `postgres:9.5.25`
 export const MONGODB_IMAGE = `mongo@${process.env.MONGODB_SHA}`
 export const MARIADB_IMAGE = `mariadb@${process.env.MARIADB_SHA}`
+export const ELASTICSEARCH_IMAGE = `elasticsearch@${process.env.ELASTICSEARCH_SHA}`